From 85e83e076b4ac4f4d1e389318de1bc8be723ddb0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Moretti?= <smoretti@unil.ch>
Date: Wed, 14 Dec 2022 16:28:39 +0100
Subject: [PATCH 001/157] Add R 4.1.0

---
 EESSI-pilot-install-software.sh | 6 ++++++
 eessi-2021.12.yml               | 4 ++++
 2 files changed, 10 insertions(+)

diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
index dc50f7fae2..ee81c47196 100755
--- a/EESSI-pilot-install-software.sh
+++ b/EESSI-pilot-install-software.sh
@@ -356,6 +356,12 @@ fail_msg="Installation of WRF failed, that's unexpected..."
 OMPI_MCA_pml=ucx UCX_TLS=tcp $EB WRF-3.9.1.1-foss-2020a-dmpar.eb -r --include-easyblocks-from-pr 2648
 check_exit_code $? "${ok_msg}" "${fail_msg}"
 
+echo ">> Installing R 4.1.0 (better be patient)..."
+ok_msg="R installed, wow!"
+fail_msg="Installation of R failed, so sad..."
+$EB --from-pr 16386 R-4.1.0-foss-2021a.eb --robot --parallel-extensions-install --experimental
+check_exit_code $? "${ok_msg}" "${fail_msg}"
+
 echo ">> Installing Nextflow 22.10.1..."
 ok_msg="Nextflow installed, the work must flow..."
 fail_msg="Installation of Nextflow failed, that's unexpected..."
diff --git a/eessi-2021.12.yml b/eessi-2021.12.yml
index 58101777ca..2b00bfe5a8 100644
--- a/eessi-2021.12.yml
+++ b/eessi-2021.12.yml
@@ -53,6 +53,10 @@ software:
          versions: 
            '3.9.1.1': 
              versionsuffix: -dmpar
+  R:
+    toolchains:
+      foss-2021a:
+        versions: '4.1.0'
   Nextflow:
     toolchains:
       SYSTEM:

From 4e2f6d046a673f7d22106c505b116e5dc2cb7de5 Mon Sep 17 00:00:00 2001
From: Michael Huebner <huebner@physik.uni-bonn.de>
Date: Tue, 20 Dec 2022 10:23:17 +0100
Subject: [PATCH 002/157] Move bash utils to new scripts dir, add new dir to
 tarball creation

---
 .github/workflows/tests_scripts.yml           |  4 +--
 EESSI-pilot-install-software.sh               |  2 +-
 ...tarball.sh => create_directory_tarballs.sh | 33 ++++++++++---------
 utils.sh => scripts/utils.sh                  |  0
 update_lmod_cache.sh                          |  2 +-
 5 files changed, 22 insertions(+), 19 deletions(-)
 rename create_init_tarball.sh => create_directory_tarballs.sh (52%)
 rename utils.sh => scripts/utils.sh (100%)

diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml
index 18fcd7b255..d224034c2a 100644
--- a/.github/workflows/tests_scripts.yml
+++ b/.github/workflows/tests_scripts.yml
@@ -6,7 +6,7 @@ on:
       - build_container.sh
       - install_software_layer.sh
       - run_in_compat_layer_env.sh
-      - utils.sh
+      - scripts/utils.sh
       - update_lmod_cache.sh
 
   pull_request:
@@ -16,7 +16,7 @@ on:
       - build_container.sh
       - install_software_layer.sh
       - run_in_compat_layer_env.sh
-      - utils.sh
+      - scripts/utils.sh
       - update_lmod_cache.sh
 permissions:
   contents: read # to fetch code (actions/checkout)
diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
index 10311b7a15..4ca553dbe7 100755
--- a/EESSI-pilot-install-software.sh
+++ b/EESSI-pilot-install-software.sh
@@ -50,7 +50,7 @@ set -- "${POSITIONAL_ARGS[@]}"
 
 TOPDIR=$(dirname $(realpath $0))
 
-source $TOPDIR/utils.sh
+source $TOPDIR/scripts/utils.sh
 
 # honor $TMPDIR if it is already defined, use /tmp otherwise
 if [ -z $TMPDIR ]; then
diff --git a/create_init_tarball.sh b/create_directory_tarballs.sh
similarity index 52%
rename from create_init_tarball.sh
rename to create_directory_tarballs.sh
index 3393285a33..c254c2c229 100755
--- a/create_init_tarball.sh
+++ b/create_directory_tarballs.sh
@@ -4,19 +4,6 @@ SOFTWARE_LAYER_TARBALL_URL=https://github.com/EESSI/software-layer/tarball/main
 
 set -eo pipefail
 
-function echo_green() {
-    echo -e "\e[32m$1\e[0m"
-}
-
-function echo_red() {
-    echo -e "\e[31m$1\e[0m"
-}
-
-function error() {
-    echo_red "ERROR: $1" >&2
-    exit 1
-}
-
 if [ $# -ne 1 ]; then
     echo "Usage: $0 <EESSI version>" >&2
     exit 1
@@ -24,13 +11,18 @@ fi
 
 version=$1
 
+TOPDIR=$(dirname $(realpath $0))
+
+source $TOPDIR/scripts/utils.sh
+
 # Check if the EESSI version number encoded in the filename
 # is valid, i.e. matches the format YYYY.DD
 if ! echo "${version}" | egrep -q '^20[0-9][0-9]\.(0[0-9]|1[0-2])$'
 then
-    error "${version} is not a valid EESSI version."
+    fatal_error "${version} is not a valid EESSI version."
 fi
 
+# Create tarball of init directory
 tartmp=$(mktemp -t -d init.XXXXX)
 mkdir "${tartmp}/${version}"
 tarname="eessi-${version}-init-$(date +%s).tar.gz"
@@ -38,9 +30,20 @@ curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --s
 source "${tartmp}/${version}/init/minimal_eessi_env"
 if [ "${EESSI_PILOT_VERSION}" != "${version}" ]
 then
-  error "Specified version ${version} does not match version ${EESSI_PILOT_VERSION} in the init files!"
+  fatal_error "Specified version ${version} does not match version ${EESSI_PILOT_VERSION} in the init files!"
 fi
 tar czf "${tarname}" -C "${tartmp}" "${version}"
 rm -rf "${tartmp}"
 
 echo_green "Done! Created tarball ${tarname}."
+
+# Create tarball of scripts directory
+# Version check has already been performed and would have caused script to exit at this point in case of problems
+tartmp=$(mktemp -t -d scripts.XXXXX)
+mkdir "${tartmp}/${version}"
+tarname="eessi-${version}-scripts-$(date +%s).tar.gz"
+curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --strip-components=1 --wildcards */scripts/
+tar czf "${tarname}" -C "${tartmp}" "${version}"
+rm -rf "${tartmp}"
+
+echo_green "Done! Created tarball ${tarname}."
diff --git a/utils.sh b/scripts/utils.sh
similarity index 100%
rename from utils.sh
rename to scripts/utils.sh
diff --git a/update_lmod_cache.sh b/update_lmod_cache.sh
index 89e2ecbeee..814bb2dae0 100755
--- a/update_lmod_cache.sh
+++ b/update_lmod_cache.sh
@@ -5,7 +5,7 @@
 
 TOPDIR=$(dirname $(realpath $0))
 
-source $TOPDIR/utils.sh
+source $TOPDIR/scripts/utils.sh
 
 if [ $# -ne 2 ]; then
     echo "Usage: $0 <path to compat layer directory> <path to software installation prefix>" >&2

From 97af63be874103f62c026160b9734a0ffb5bf146 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Moretti?= <smoretti@unil.ch>
Date: Tue, 20 Dec 2022 16:34:05 +0100
Subject: [PATCH 003/157] Move to a less problematic R 4.1.0 PR

---
 EESSI-pilot-install-software.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
index ee81c47196..a33ebc9e54 100755
--- a/EESSI-pilot-install-software.sh
+++ b/EESSI-pilot-install-software.sh
@@ -359,7 +359,7 @@ check_exit_code $? "${ok_msg}" "${fail_msg}"
 echo ">> Installing R 4.1.0 (better be patient)..."
 ok_msg="R installed, wow!"
 fail_msg="Installation of R failed, so sad..."
-$EB --from-pr 16386 R-4.1.0-foss-2021a.eb --robot --parallel-extensions-install --experimental
+$EB --from-pr 16011 R-4.1.0-foss-2021a.eb --robot --parallel-extensions-install --experimental
 check_exit_code $? "${ok_msg}" "${fail_msg}"
 
 echo ">> Installing Nextflow 22.10.1..."

From 33a34203b40f838fb9f8bc6c1f27ee5c804ad2bb Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 20 Dec 2022 17:54:13 +0100
Subject: [PATCH 004/157] install X11 dependency for R 4.1.0 via PR to pick up
 patch that fixes installation problem for libXfont2 component

---
 EESSI-pilot-install-software.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
index a33ebc9e54..b68ff1c93d 100755
--- a/EESSI-pilot-install-software.sh
+++ b/EESSI-pilot-install-software.sh
@@ -359,7 +359,7 @@ check_exit_code $? "${ok_msg}" "${fail_msg}"
 echo ">> Installing R 4.1.0 (better be patient)..."
 ok_msg="R installed, wow!"
 fail_msg="Installation of R failed, so sad..."
-$EB --from-pr 16011 R-4.1.0-foss-2021a.eb --robot --parallel-extensions-install --experimental
+$EB --from-pr 14821 X11-20210518-GCCcore-10.3.0.eb -r && $EB --from-pr 16011 R-4.1.0-foss-2021a.eb --robot --parallel-extensions-install --experimental
 check_exit_code $? "${ok_msg}" "${fail_msg}"
 
 echo ">> Installing Nextflow 22.10.1..."

From 679fe863a9c42041aba19fbf66923ecc73db3dca Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 30 Dec 2022 05:19:36 +0100
Subject: [PATCH 005/157] use single source file for pilot version and
 repository

- adds a new configuration file init/eessi_defaults which sets
  EESSI_PILOT_VERSION to a default or EESSI_PILOT_VERSION_OVERRIDE
  and EESSI_CVMFS_REPO to a default or EESSI_CVMFS_REPO_OVERRIDE
- existing init files and scripts that used hardcoded values are
  adjusted to source the new configuration file
- a comment in EESSI-pilot-install-software.sh is updated that not
  a hardcoded version but the one from init/eessi_defaults is used
- a simple test is added which checks if the default values in
  init/eessi_defaults correspond to values used in README.md
---
 .github/workflows/tests_readme.yml | 31 ++++++++++++++++++++++++++++++
 EESSI-pilot-install-software.sh    |  2 +-
 build_container.sh                 | 10 ++++++++--
 create_tarball.sh                  |  5 ++++-
 init/eessi_defaults                | 13 +++++++++++++
 init/minimal_eessi_env             | 10 ++++++++--
 install_software_layer.sh          |  3 ++-
 run_in_compat_layer_env.sh         |  7 ++++++-
 8 files changed, 73 insertions(+), 8 deletions(-)
 create mode 100644 .github/workflows/tests_readme.yml
 create mode 100644 init/eessi_defaults

diff --git a/.github/workflows/tests_readme.yml b/.github/workflows/tests_readme.yml
new file mode 100644
index 0000000000..29dedcc3ff
--- /dev/null
+++ b/.github/workflows/tests_readme.yml
@@ -0,0 +1,31 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Tests for consistency of README.md
+on:
+  push:
+    paths:
+      - README.md
+      - init/eessi_defaults
+
+  pull_request:
+    branches:
+      - main
+    paths:
+      - README.md
+      - init/eessi_defaults
+permissions:
+  contents: read # to fetch code (actions/checkout)
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: verify if README.md is consistent with EESSI_PILOT_VERSION from init/eessi_defaults
+      run: |
+        source init/eessi_defaults
+        grep "${EESSI_PILOT_VERSION}" README.md
+
+    - name: verify if README.md is consistent with EESSI_CVMFS_REPO from init/eessi_defaults
+      run: |
+        source init/eessi_defaults
+        grep "${EESSI_CVMFS_REPO}" README.md
diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
index 10311b7a15..d4d4fd28e8 100755
--- a/EESSI-pilot-install-software.sh
+++ b/EESSI-pilot-install-software.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Script to install EESSI pilot software stack (version 2021.12)
+# Script to install EESSI pilot software stack (version set through init/eessi_defaults)
 
 # see example parsing of command line arguments at
 #   https://wiki.bash-hackers.org/scripting/posparams#using_a_while_loop
diff --git a/build_container.sh b/build_container.sh
index d7e7256b4e..e087b9f7be 100755
--- a/build_container.sh
+++ b/build_container.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+base_dir=$(dirname $(realpath $0))
+
 BUILD_CONTAINER="docker://ghcr.io/eessi/build-node:debian10"
 
 if [ $# -lt 2 ]; then
@@ -39,9 +41,13 @@ if [ -z $SINGULARITY_HOME ]; then
     export SINGULARITY_HOME="$EESSI_TMPDIR/home:/home/$USER"
 fi
 
+source ${base_dir}/init/eessi_defaults
+# strip "/cvmfs/" from default setting
+repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//}
+
 # set environment variables for fuse mounts in Singularity container
-export EESSI_PILOT_READONLY="container:cvmfs2 pilot.eessi-hpc.org /cvmfs_ro/pilot.eessi-hpc.org"
-export EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs -o lowerdir=/cvmfs_ro/pilot.eessi-hpc.org -o upperdir=$EESSI_TMPDIR/overlay-upper -o workdir=$EESSI_TMPDIR/overlay-work /cvmfs/pilot.eessi-hpc.org"
+export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}"
+export EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs -o lowerdir=/cvmfs_ro/${repo_name} -o upperdir=$EESSI_TMPDIR/overlay-upper -o workdir=$EESSI_TMPDIR/overlay-work ${EESSI_CVMFS_REPO}"
 
 if [ "$SHELL_OR_RUN" == "shell" ]; then
     # start shell in Singularity container, with EESSI repository mounted with writable overlay
diff --git a/create_tarball.sh b/create_tarball.sh
index 619d6ea2e1..142df60a94 100755
--- a/create_tarball.sh
+++ b/create_tarball.sh
@@ -2,6 +2,8 @@
 
 set -e
 
+base_dir=$(dirname $(realpath $0))
+
 if [ $# -ne 4 ]; then
     echo "ERROR: Usage: $0 <EESSI tmp dir (example: /tmp/$USER/EESSI)> <pilot version (example: 2021.03)> <CPU arch subdir (example: x86_64/amd/zen2)> <path to tarball>" >&2
     exit 1
@@ -15,7 +17,8 @@ tmpdir=`mktemp -d`
 echo ">> tmpdir: $tmpdir"
 
 os="linux"
-cvmfs_repo="/cvmfs/pilot.eessi-hpc.org"
+source ${base_dir}/init/eessi_defaults
+cvmfs_repo=${EESSI_CVMFS_REPO}
 
 software_dir="${cvmfs_repo}/versions/${pilot_version}/software/${os}/${cpu_arch_subdir}"
 if [ ! -d ${software_dir} ]; then
diff --git a/init/eessi_defaults b/init/eessi_defaults
new file mode 100644
index 0000000000..1b5ce07fb1
--- /dev/null
+++ b/init/eessi_defaults
@@ -0,0 +1,13 @@
+# define default values for some EESSI_* environment variables
+#
+# This file is part of the EESSI software layer,
+# see https://github.com/EESSI/software-layer
+#
+# author: Thomas Roeblitz (@trz42)
+#
+# license: GPLv2
+#
+
+export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/pilot.eessi-hpc.org}"
+export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2021.12}"
+
diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env
index 39478d5c4c..b7cb7c5e9e 100644
--- a/init/minimal_eessi_env
+++ b/init/minimal_eessi_env
@@ -1,7 +1,13 @@
 # define minimal EESSI environment, without relying on external scripts
+#
+# this script is *sourced*, not executed, so can't rely on $0 to determine path to self
+# $BASH_SOURCE points to correct path, see also http://mywiki.wooledge.org/BashFAQ/028
+EESSI_INIT_DIR_PATH=$(dirname $(realpath $BASH_SOURCE))
+
+# set up defaults: EESSI_CVMFS_REPO, EESSI_PILOT_VERSION
+#   script takes *_OVERRIDEs into account
+source ${EESSI_INIT_DIR_PATH}/eessi_defaults
 
-export EESSI_CVMFS_REPO="/cvmfs/pilot.eessi-hpc.org"
-export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2021.12}"
 export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_PILOT_VERSION
 
 if [[ $(uname -s) == 'Linux' ]]; then
diff --git a/install_software_layer.sh b/install_software_layer.sh
index e0b2f785d5..bf3006a4a0 100755
--- a/install_software_layer.sh
+++ b/install_software_layer.sh
@@ -1,3 +1,4 @@
 #!/bin/bash
-export EESSI_PILOT_VERSION='2021.12'
+base_dir=$(dirname $(realpath $0))
+source ${base_dir}/init/eessi_defaults
 ./run_in_compat_layer_env.sh ./EESSI-pilot-install-software.sh "$@"
diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh
index 118aac65d5..41a7be1ca0 100755
--- a/run_in_compat_layer_env.sh
+++ b/run_in_compat_layer_env.sh
@@ -1,9 +1,14 @@
 #!/bin/bash
+
+base_dir=$(dirname $(realpath $0))
+source ${base_dir}/init/eessi_defaults
+
+BUILD_CONTAINER="docker://ghcr.io/eessi/build-node:debian10"
 if [ -z $EESSI_PILOT_VERSION ]; then
     echo "ERROR: \$EESSI_PILOT_VERSION must be set!" >&2
     exit 1
 fi
-EESSI_COMPAT_LAYER_DIR="/cvmfs/pilot.eessi-hpc.org/versions/${EESSI_PILOT_VERSION}/compat/linux/$(uname -m)"
+EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_PILOT_VERSION}/compat/linux/$(uname -m)"
 if [ ! -d ${EESSI_COMPAT_LAYER_DIR} ]; then
     echo "ERROR: ${EESSI_COMPAT_LAYER_DIR} does not exist!" >&2
     exit 1

From 2caad9bc9cc99544542725894c7ee11c8b6ea701 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 1 Jan 2023 20:42:40 +0100
Subject: [PATCH 006/157] initial version of unified script

- should support existing ways to access EESSI
- no additional features implemented yet (--help may hint at new features)
---
 eessi_container.sh | 311 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 311 insertions(+)
 create mode 100755 eessi_container.sh

diff --git a/eessi_container.sh b/eessi_container.sh
new file mode 100755
index 0000000000..9219581582
--- /dev/null
+++ b/eessi_container.sh
@@ -0,0 +1,311 @@
+#!/bin/bash
+#
+# unified script to access EESSI in different scenarios: read-only
+# for just using EESSI, read & write for building software to be
+# added to the software stack
+#
+# This script is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Thomas Roeblitz (@trz42)
+#
+# license: GPLv2
+#
+
+# script overview
+# -. initial settings & exit codes
+# 0. parse args
+# 1. check if argument values are valid
+# 2. set up local disk/tmp
+# 3. set up common vars and directories
+# 4. set up vars specific to a scenario
+# 5. initialize local disk/tmp from previous run if provided
+# 6. run container
+
+# -. initial settings & exit codes
+base_dir=$(dirname $(realpath $0))
+
+# functions
+function echo_red() {
+    echo -e "\e[31m$1\e[0m"
+}
+
+function fatal_error() {
+    echo_red "ERROR: ${1}" >&2
+    exit ${2}
+}
+
+# exit codes: bitwise shift codes to allow for combination of exit codes
+ANY_ERROR_EXITCODE=1
+CMDLINE_ARG_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 1))
+ACCESS_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 2))
+CONTAINER_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 3))
+LOCAL_DISK_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 4))
+MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5))
+PREVIOUS_RUN_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6))
+REPOSITORY_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7))
+HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 8))
+HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9))
+RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10))
+
+# CernVM-FS settings
+CVMFS_VAR_LIB="var-lib-cvmfs"
+CVMFS_VAR_RUN="var-run-cvmfs"
+
+
+# 0. parse args
+#    see example parsing of command line arguments at
+#    https://wiki.bash-hackers.org/scripting/posparams#using_a_while_loop
+#    https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash
+
+display_help() {
+  echo "usage: $0 [OPTIONS] [SCRIPT]"
+  echo " OPTIONS:"
+  echo "  -a | --access {ro,rw}    -  ro (read-only), rw (read & write) [default: ro]"
+  echo "  -c | --container IMAGE   -  image file or URL defining the container to use"
+  echo "                           [default: docker://ghcr.io/eessi/build-node:debian10]"
+  echo "  -d | --dry-run           -  run script except for executing the container,"
+  echo "                              print information about setup [default: false]"
+  echo "  -h | --help              -  display this usage information [default: false]"
+  echo "  -i | --info              -  display configured repositories [default: false]"
+  echo "  -l | --local-disk DIR    -  directory space on local machine (used for"
+  echo "                              temporary data) [default: 1. TMPDIR, 2. /tmp]"
+  echo "  -m | --mode {shell,run}  -  shell (launch interactive shell)"
+  echo "                              run (run a script) [default: shell]"
+  echo "  -p | --previous-run PRUN -  init local disk with data from previous run"
+  echo "                              format is PATH[:TAR/ZIP] where PATH is pointing"
+  echo "                              to the previously used local disk, and TAR/ZIP"
+  echo "                              is used to initialize the local disk if PATH"
+  echo "                              doesn't exist currently; if PATH exists and"
+  echo "                              a TAR/ZIP is provided an error is reported"
+  echo "                              [default: not set]"
+  echo "  -r | --repository CFG    -  configuration file or identifier defining the"
+  echo "                              repository to use [default: EESSI-pilot]"
+  echo "  -x | --http-proxy URL    -  provides URL for the env variable http_proxy"
+  echo "                              [default: not set]"
+  echo "  -y | --https-proxy URL   -  provides URL for the env variable https_proxy"
+  echo "                              [default: not set]"
+  echo
+  echo " If value for --mode is 'run', the SCRIPT provided is executed."
+}
+
+# set defaults for command line arguments
+ACCESS="ro"
+CONTAINER="docker://ghcr.io/eessi/build-node:debian10"
+DRY_RUN=0
+INFO=0
+LOCAL_DISK=
+MODE="shell"
+PREVIOUS_RUN=
+REPOSITORY="EESSI-pilot"
+HTTP_PROXY=
+HTTPS_PROXY=
+RUN_SCRIPT_AND_ARGS=
+
+POSITIONAL_ARGS=()
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -a|--access)
+      ACCESS="$2"
+      shift 2
+      ;;
+    -c|--container)
+      CONTAINER="$2"
+      shift 2
+      ;;
+    -d|--dry-run)
+      DRY_RUN=1
+      ;;
+    -h|--help)
+      display_help
+      exit 0
+      ;;
+    -i|--info)
+      INFO=1
+      ;;
+    -l|--local-disk)
+      LOCAL_DISK="$2"
+      #EESSI_TMPDIR="$2"
+      shift 2
+      ;;
+    -m|--mode)
+      MODE="$2"
+      shift 2
+      ;;
+    -p|--previous-run)
+      PREVIOUS_RUN="$2"
+      shift 2
+      ;;
+    -x|--http-proxy)
+      HTTP_PROXY="$2"
+      export http_proxy=${HTTP_PROXY}
+      shift 2
+      ;;
+    -y|--https-proxy)
+      HTTPS_PROXY="$2"
+      export https_proxy=${HTTPS_PROXY}
+      shift 2
+      ;;
+    -*|--*)
+      fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}"
+      ;;
+    *)  # No more options
+      POSITIONAL_ARGS+=("$1") # save positional arg
+      shift
+      ;;
+  esac
+done
+
+set -- "${POSITIONAL_ARGS[@]}"
+
+
+# 1. check if argument values are valid
+# (arg -a|--access) check if ACCESS is supported
+if [[ ${ACCESS} != "ro" -a ${ACCESS} != "rw" ]]; then
+  fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}"
+fi
+
+# TODO (arg -c|--container) check container (is it a file or URL & access those)
+# CONTAINER_ERROR_EXITCODE
+
+# TODO (arg -l|--local-disk) check if it exists, if user has write permission,
+#      if it contains no data, etc.
+# LOCAL_DISK_ERROR_EXITCODE
+
+# (arg -m|--mode) check if MODE is known
+if [[ ${MODE} != "shell" -a ${MODE} != "run" ]]; then
+  fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}"
+fi
+
+# TODO (arg -p|--previous-run) check if it exists, if user has read permission,
+#      if it contains data from a previous run
+# PREVIOUS_RUN_ERROR_EXITCODE
+
+# TODO (arg -r|--repository) check if repository is known
+# REPOSITORY_UNKNOWN_EXITCODE
+
+# TODO (arg -x|--http-proxy) check if http proxy is accessible
+# HTTP_PROXY_ERROR_EXITCODE
+
+# TODO (arg -y|--https-proxy) check if https proxy is accessible
+# HTTPS_PROXY_ERROR_EXITCODE
+
+# check if a script is provided if mode is 'run'
+if [[ "${MODE}" == "run" ]]; then
+  if [[ $# -eq 0 ]]; then
+    fatal_error "no command specified to run?!" "${RUN_SCRIPT_MISSING_EXITCODE}"
+  else
+    RUN_SCRIPT_AND_ARGS=$@
+  fi
+fi
+
+
+# 2. set up local disk/tmp
+# as location for temporary data use in the following order
+#   a. command line argument -l|--local_disk
+#   b. env var TMPDIR
+#   c. /tmp
+# note, we ensure that (a) takes precedence by setting TMPDIR to LOCAL_DISK
+#     if LOCAL_DISK is not empty
+# note, (b) & (c) are automatically ensured by using mktemp -d to create
+#     a temporary directory
+# note, if previous run is used the name of the temporary directory
+#     should be identical to previous run, ie, then we don't create a new
+#     temporary directory
+if [[ ! -z ${LOCAL_DISK} ]]; then
+  TMPDIR=${LOCAL_DISK}
+fi
+if [[ ! -z ${TMPDIR} ]]; then
+  # TODO check if TMPDIR already exists
+  EESSI_LOCAL_DISK=${TMPDIR}
+fi
+if [[ -z ${TMPDIR} ]]; then
+  # mktemp falls back to using /tmp if TMPDIR is empty
+  # TODO check if /tmp is writable, large enough and usable (different
+  #      features for ro-access and rw-access)
+  echo "skipping sanity checks for /tmp"
+fi
+EESSI_LOCAL_DISK=$(mktemp -d eessi.XXXXXXXXXX)
+echo "Using ${EESSI_LOCAL_DISK} as parent for temporary directories..."
+
+
+# 3. set up common vars and directories
+#    directory structure should be:
+#      ${EESSI_LOCAL_DISK}
+#      |-singularity_cache
+#      |-${CVMFS_VAR_LIB}
+#      |-${CVMFS_VAR_RUN}
+
+source ${base_dir}/init/eessi_defaults
+
+# configure Singularity
+export SINGULARITY_CACHEDIR=${EESSI_LOCAL_DISK}/singularity_cache
+mkdir -p ${SINGULARITY_CACHEDIR}
+[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}"
+
+# set env vars and create directories for CernVM-FS
+EESSI_CVMFS_VAR_LIB=${EESSI_LOCAL_DISK}/${CVMFS_VAR_LIB}
+EESSI_CVMFS_VAR_RUN=${EESSI_LOCAL_DISK}/${CVMFS_VAR_RUN}
+mkdir -p ${EESSI_CVMFS_VAR_LIB}
+mkdir -p ${EESSI_CVMFS_VAR_RUN}
+[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_LIB=${EESSI_CVMFS_VAR_LIB}"
+[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_RUN=${EESSI_CVMFS_VAR_RUN}"
+
+# tmp dir for EESSI
+EESSI_TMPDIR=${EESSI_LOCAL_DISK}/tmp
+mkdir -p ${EESSI_TMPDIR}
+[[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}"
+
+# allow that SINGULARITY_HOME is defined before script is run
+if [[ -z ${SINGULARITY_HOME} ]]; then
+  export SINGULARITY_HOME="${EESSI_LOCAL_DISK}/home:/home/${USER}"
+  mkdir -p ${EESSI_LOCAL_DISK}/home
+  [[ ${INFO} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}"
+fi
+
+# define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined)
+BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvfms,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs"
+BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp"
+
+
+# 4. set up vars and dirs specific to a scenario
+
+# strip "/cvmfs/" from default setting
+repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//}
+
+if [[ "${ACCESS}" == "ro" ]]; then
+  export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} ${EESSI_CVMFS_REPO}"
+  export EESSI_FUSE_MOUNTS="--fusemount ${EESSI_PILOT_READONLY}"
+fi
+
+if [[ "${ACCESS}" == "rw" ]]; then
+  EESSI_CVMFS_OVERLAY_UPPER=${EESSI_LOCAL_DISK}/overlay-upper
+  EESSI_CVMFS_OVERLAY_WORK=${EESSI_LOCAL_DISK}/overlay-work
+  mkdir -p ${EESSI_CMVFS_OVERLAY_UPPER}
+  mkdir -p ${EESSI_CMVFS_OVERLAY_WORK}
+  [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}"
+  [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}"
+
+  # set environment variables for fuse mounts in Singularity container
+  export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}"
+  EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs"
+  EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}"
+  EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${EESSI_CVMFS_OVERLAY_UPPER}"
+  EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${EESSI_CVMFS_OVERLAY_WORK}"
+  EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}"
+  export EESSI_PILOT_WRITABLE_OVERLAY
+
+  EESSI_FUSE_MOUNTS="--fusemount ${EESSI_PILOT_READONLY}"
+  EESSI_FUSE_MOUNTS+=" --fusemount ${EESSI_PILOT_WRITABLE_OVERLAY}"
+  export EESSI_FUSE_MOUNTS
+fi
+
+
+# 5. initialize local disk/tmp from previous run if provided
+
+
+# 6. run container
+echo "Launching container with command (next line):"
+echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}"
+singularity ${MODE} ${EESSI_FUSE_MOUNTS} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}

From 6f904cee6e6aa9d83e79e0bf7eabbc329885561b Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 1 Jan 2023 20:54:01 +0100
Subject: [PATCH 007/157] fix test syntax

---
 eessi_container.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 9219581582..273d3bb297 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -162,7 +162,7 @@ set -- "${POSITIONAL_ARGS[@]}"
 
 # 1. check if argument values are valid
 # (arg -a|--access) check if ACCESS is supported
-if [[ ${ACCESS} != "ro" -a ${ACCESS} != "rw" ]]; then
+if [[ "${ACCESS}" != "ro" && "${ACCESS}" != "rw" ]]; then
   fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}"
 fi
 
@@ -174,7 +174,7 @@ fi
 # LOCAL_DISK_ERROR_EXITCODE
 
 # (arg -m|--mode) check if MODE is known
-if [[ ${MODE} != "shell" -a ${MODE} != "run" ]]; then
+if [[ "${MODE}" != "shell" && "${MODE}" != "run" ]]; then
   fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}"
 fi
 

From d6aae8d03247f2fe0e571989070ff94764dfdcd6 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 1 Jan 2023 21:44:53 +0100
Subject: [PATCH 008/157] use array for EESSI_FUSE_MOUNTS, fix var name typo

---
 eessi_container.sh | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 273d3bb297..23d6cd4bf5 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -274,31 +274,30 @@ BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp"
 # strip "/cvmfs/" from default setting
 repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//}
 
+declare -a EESSI_FUSE_MOUNTS=()
 if [[ "${ACCESS}" == "ro" ]]; then
   export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} ${EESSI_CVMFS_REPO}"
-  export EESSI_FUSE_MOUNTS="--fusemount ${EESSI_PILOT_READONLY}"
+  EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_READONLY}")
 fi
 
 if [[ "${ACCESS}" == "rw" ]]; then
   EESSI_CVMFS_OVERLAY_UPPER=${EESSI_LOCAL_DISK}/overlay-upper
   EESSI_CVMFS_OVERLAY_WORK=${EESSI_LOCAL_DISK}/overlay-work
-  mkdir -p ${EESSI_CMVFS_OVERLAY_UPPER}
-  mkdir -p ${EESSI_CMVFS_OVERLAY_WORK}
+  mkdir -p ${EESSI_CVMFS_OVERLAY_UPPER}
+  mkdir -p ${EESSI_CVMFS_OVERLAY_WORK}
   [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}"
   [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}"
 
   # set environment variables for fuse mounts in Singularity container
-  export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}"
+  EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}"
+  EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_READONLY}")
+
   EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs"
   EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}"
   EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${EESSI_CVMFS_OVERLAY_UPPER}"
   EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${EESSI_CVMFS_OVERLAY_WORK}"
   EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}"
-  export EESSI_PILOT_WRITABLE_OVERLAY
-
-  EESSI_FUSE_MOUNTS="--fusemount ${EESSI_PILOT_READONLY}"
-  EESSI_FUSE_MOUNTS+=" --fusemount ${EESSI_PILOT_WRITABLE_OVERLAY}"
-  export EESSI_FUSE_MOUNTS
+  EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_WRITABLE_OVERLAY}")
 fi
 
 
@@ -307,5 +306,5 @@ fi
 
 # 6. run container
 echo "Launching container with command (next line):"
-echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}"
-singularity ${MODE} ${EESSI_FUSE_MOUNTS} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}
+echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}"
+singularity ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}

From 4194421b5075bfc8714a5d6d0f3cd554d35992aa Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 1 Jan 2023 21:46:01 +0100
Subject: [PATCH 009/157] add missing shift operations

---
 eessi_container.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/eessi_container.sh b/eessi_container.sh
index 23d6cd4bf5..fcc5cc1e31 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -116,6 +116,7 @@ while [[ $# -gt 0 ]]; do
       ;;
     -d|--dry-run)
       DRY_RUN=1
+      shift 1
       ;;
     -h|--help)
       display_help
@@ -123,6 +124,7 @@ while [[ $# -gt 0 ]]; do
       ;;
     -i|--info)
       INFO=1
+      shift 1
       ;;
     -l|--local-disk)
       LOCAL_DISK="$2"

From 8b2f7301fc9cf7e2b583397a4f603ab74f4efcc0 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 1 Jan 2023 22:10:09 +0100
Subject: [PATCH 010/157] fixing creation of temporary directory

---
 eessi_container.sh | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index fcc5cc1e31..3d64585d0d 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -210,17 +210,20 @@ fi
 #   c. /tmp
 # note, we ensure that (a) takes precedence by setting TMPDIR to LOCAL_DISK
 #     if LOCAL_DISK is not empty
-# note, (b) & (c) are automatically ensured by using mktemp -d to create
-#     a temporary directory
+# note, (b) & (c) are automatically ensured by using 'mktemp -d --tmpdir' to
+#     create a temporary directory
 # note, if previous run is used the name of the temporary directory
 #     should be identical to previous run, ie, then we don't create a new
 #     temporary directory
 if [[ ! -z ${LOCAL_DISK} ]]; then
   TMPDIR=${LOCAL_DISK}
+  # mktemp fails if TMPDIR does not exist, so let's create it
+  mkdir -p ${TMPDIR}
 fi
 if [[ ! -z ${TMPDIR} ]]; then
   # TODO check if TMPDIR already exists
-  EESSI_LOCAL_DISK=${TMPDIR}
+  # mktemp fails if TMPDIR does not exist, so let's create it
+  mkdir -p ${TMPDIR}
 fi
 if [[ -z ${TMPDIR} ]]; then
   # mktemp falls back to using /tmp if TMPDIR is empty
@@ -228,7 +231,7 @@ if [[ -z ${TMPDIR} ]]; then
   #      features for ro-access and rw-access)
   echo "skipping sanity checks for /tmp"
 fi
-EESSI_LOCAL_DISK=$(mktemp -d eessi.XXXXXXXXXX)
+EESSI_LOCAL_DISK=$(mktemp -d --tmpdir eessi.XXXXXXXXXX)
 echo "Using ${EESSI_LOCAL_DISK} as parent for temporary directories..."
 
 

From 6aa5cac8d7335e47120c5504de2fb8e1aa86f95e Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 1 Jan 2023 22:32:10 +0100
Subject: [PATCH 011/157] fix BIND typo, define SINGULARITY_BIND

---
 eessi_container.sh | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 3d64585d0d..0425d8759b 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -270,8 +270,9 @@ if [[ -z ${SINGULARITY_HOME} ]]; then
 fi
 
 # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined)
-BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvfms,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs"
+BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs"
 BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp"
+[[ ${INFO} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}"
 
 
 # 4. set up vars and dirs specific to a scenario
@@ -310,6 +311,14 @@ fi
 
 
 # 6. run container
+# final settings
+if [[ -z ${SINGULARITY_BIND} ]]; then
+    export SINGULARITY_BIND="${BIND_PATHS}"
+else
+    export SINGULARITY_BIND="${SINGULARITY_BIND},${BIND_PATHS}"
+fi
+[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}"
+
 echo "Launching container with command (next line):"
 echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}"
 singularity ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}

From 3a2bf51687bb1c9b8e39518e2481939e84dfc751 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 1 Jan 2023 23:48:28 +0100
Subject: [PATCH 012/157] fix tmp dir for overlay fs

---
 eessi_container.sh | 35 +++++++++++++++++++----------------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 0425d8759b..1be3b566db 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -241,31 +241,34 @@ echo "Using ${EESSI_LOCAL_DISK} as parent for temporary directories..."
 #      |-singularity_cache
 #      |-${CVMFS_VAR_LIB}
 #      |-${CVMFS_VAR_RUN}
+#      |-overlay-upper
+#      |-overlay-work
+#      |-home
 
 source ${base_dir}/init/eessi_defaults
 
+# tmp dir for EESSI
+EESSI_TMPDIR=${EESSI_LOCAL_DISK}
+mkdir -p ${EESSI_TMPDIR}
+[[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}"
+
 # configure Singularity
-export SINGULARITY_CACHEDIR=${EESSI_LOCAL_DISK}/singularity_cache
+export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache
 mkdir -p ${SINGULARITY_CACHEDIR}
 [[ ${INFO} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}"
 
 # set env vars and create directories for CernVM-FS
-EESSI_CVMFS_VAR_LIB=${EESSI_LOCAL_DISK}/${CVMFS_VAR_LIB}
-EESSI_CVMFS_VAR_RUN=${EESSI_LOCAL_DISK}/${CVMFS_VAR_RUN}
+EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB}
+EESSI_CVMFS_VAR_RUN=${EESSI_TMPDIR}/${CVMFS_VAR_RUN}
 mkdir -p ${EESSI_CVMFS_VAR_LIB}
 mkdir -p ${EESSI_CVMFS_VAR_RUN}
 [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_LIB=${EESSI_CVMFS_VAR_LIB}"
 [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_RUN=${EESSI_CVMFS_VAR_RUN}"
 
-# tmp dir for EESSI
-EESSI_TMPDIR=${EESSI_LOCAL_DISK}/tmp
-mkdir -p ${EESSI_TMPDIR}
-[[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}"
-
 # allow that SINGULARITY_HOME is defined before script is run
 if [[ -z ${SINGULARITY_HOME} ]]; then
-  export SINGULARITY_HOME="${EESSI_LOCAL_DISK}/home:/home/${USER}"
-  mkdir -p ${EESSI_LOCAL_DISK}/home
+  export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}"
+  mkdir -p ${EESSI_TMPDIR}/home
   [[ ${INFO} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}"
 fi
 
@@ -287,10 +290,10 @@ if [[ "${ACCESS}" == "ro" ]]; then
 fi
 
 if [[ "${ACCESS}" == "rw" ]]; then
-  EESSI_CVMFS_OVERLAY_UPPER=${EESSI_LOCAL_DISK}/overlay-upper
-  EESSI_CVMFS_OVERLAY_WORK=${EESSI_LOCAL_DISK}/overlay-work
-  mkdir -p ${EESSI_CVMFS_OVERLAY_UPPER}
-  mkdir -p ${EESSI_CVMFS_OVERLAY_WORK}
+  EESSI_CVMFS_OVERLAY_UPPER=/tmp/overlay-upper
+  EESSI_CVMFS_OVERLAY_WORK=/tmp/overlay-work
+  mkdir -p ${EESSI_TMPDIR}/overlay-upper
+  mkdir -p ${EESSI_TMPDIR}/overlay-work
   [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}"
   [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}"
 
@@ -300,8 +303,8 @@ if [[ "${ACCESS}" == "rw" ]]; then
 
   EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs"
   EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}"
-  EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${EESSI_CVMFS_OVERLAY_UPPER}"
-  EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${EESSI_CVMFS_OVERLAY_WORK}"
+  EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=/tmp/overlay-upper"
+  EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=/tmp/overlay-work"
   EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}"
   EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_WRITABLE_OVERLAY}")
 fi

From 3fbea5e4faa3f4acbe31108fe8f08e2a5f088405 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 2 Jan 2023 18:53:53 +0100
Subject: [PATCH 013/157] fixing export bug, improving readability

- added missing export for TMPDIR
- moved functions to utils.sh
- renamed local-disk/LOCAL_DISK to host-storage/HOST_STORAGE
---
 eessi_container.sh | 58 ++++++++++++++++++-----------------------
 utils.sh           | 65 +++++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 89 insertions(+), 34 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 1be3b566db..b597b17647 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -16,37 +16,29 @@
 # -. initial settings & exit codes
 # 0. parse args
 # 1. check if argument values are valid
-# 2. set up local disk/tmp
+# 2. set up host storage/tmp
 # 3. set up common vars and directories
 # 4. set up vars specific to a scenario
-# 5. initialize local disk/tmp from previous run if provided
+# 5. initialize host storage/tmp from previous run if provided
 # 6. run container
 
 # -. initial settings & exit codes
 base_dir=$(dirname $(realpath $0))
 
-# functions
-function echo_red() {
-    echo -e "\e[31m$1\e[0m"
-}
-
-function fatal_error() {
-    echo_red "ERROR: ${1}" >&2
-    exit ${2}
-}
+source ${base_dir}/utils.sh
 
 # exit codes: bitwise shift codes to allow for combination of exit codes
-ANY_ERROR_EXITCODE=1
+# ANY_ERROR_EXITCODE is sourced from ${base_dir}/utils.sh
 CMDLINE_ARG_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 1))
 ACCESS_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 2))
 CONTAINER_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 3))
-LOCAL_DISK_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 4))
+HOST_STORAGE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 4))
 MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5))
 PREVIOUS_RUN_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6))
 REPOSITORY_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7))
-HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 8))
-HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9))
-RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10))
+HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9))
+HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10))
+RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11))
 
 # CernVM-FS settings
 CVMFS_VAR_LIB="var-lib-cvmfs"
@@ -68,7 +60,7 @@ display_help() {
   echo "                              print information about setup [default: false]"
   echo "  -h | --help              -  display this usage information [default: false]"
   echo "  -i | --info              -  display configured repositories [default: false]"
-  echo "  -l | --local-disk DIR    -  directory space on local machine (used for"
+  echo "  -l | --host-storage DIR  -  directory space on host machine (used for"
   echo "                              temporary data) [default: 1. TMPDIR, 2. /tmp]"
   echo "  -m | --mode {shell,run}  -  shell (launch interactive shell)"
   echo "                              run (run a script) [default: shell]"
@@ -94,7 +86,7 @@ ACCESS="ro"
 CONTAINER="docker://ghcr.io/eessi/build-node:debian10"
 DRY_RUN=0
 INFO=0
-LOCAL_DISK=
+HOST_STORAGE=
 MODE="shell"
 PREVIOUS_RUN=
 REPOSITORY="EESSI-pilot"
@@ -126,8 +118,8 @@ while [[ $# -gt 0 ]]; do
       INFO=1
       shift 1
       ;;
-    -l|--local-disk)
-      LOCAL_DISK="$2"
+    -l|--host-storage)
+      HOST_STORAGE="$2"
       #EESSI_TMPDIR="$2"
       shift 2
       ;;
@@ -171,9 +163,9 @@ fi
 # TODO (arg -c|--container) check container (is it a file or URL & access those)
 # CONTAINER_ERROR_EXITCODE
 
-# TODO (arg -l|--local-disk) check if it exists, if user has write permission,
+# TODO (arg -l|--host-storage) check if it exists, if user has write permission,
 #      if it contains no data, etc.
-# LOCAL_DISK_ERROR_EXITCODE
+# HOST_STORAGE_ERROR_EXITCODE
 
 # (arg -m|--mode) check if MODE is known
 if [[ "${MODE}" != "shell" && "${MODE}" != "run" ]]; then
@@ -203,20 +195,20 @@ if [[ "${MODE}" == "run" ]]; then
 fi
 
 
-# 2. set up local disk/tmp
+# 2. set up host storage/tmp
 # as location for temporary data use in the following order
-#   a. command line argument -l|--local_disk
+#   a. command line argument -l|--host-storage
 #   b. env var TMPDIR
 #   c. /tmp
-# note, we ensure that (a) takes precedence by setting TMPDIR to LOCAL_DISK
-#     if LOCAL_DISK is not empty
+# note, we ensure that (a) takes precedence by setting TMPDIR to HOST_STORAGE
+#     if HOST_STORAGE is not empty
 # note, (b) & (c) are automatically ensured by using 'mktemp -d --tmpdir' to
 #     create a temporary directory
 # note, if previous run is used the name of the temporary directory
 #     should be identical to previous run, ie, then we don't create a new
 #     temporary directory
-if [[ ! -z ${LOCAL_DISK} ]]; then
-  TMPDIR=${LOCAL_DISK}
+if [[ ! -z ${HOST_STORAGE} ]]; then
+  export TMPDIR=${HOST_STORAGE}
   # mktemp fails if TMPDIR does not exist, so let's create it
   mkdir -p ${TMPDIR}
 fi
@@ -231,13 +223,13 @@ if [[ -z ${TMPDIR} ]]; then
   #      features for ro-access and rw-access)
   echo "skipping sanity checks for /tmp"
 fi
-EESSI_LOCAL_DISK=$(mktemp -d --tmpdir eessi.XXXXXXXXXX)
-echo "Using ${EESSI_LOCAL_DISK} as parent for temporary directories..."
+EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX)
+echo "Using ${EESSI_HOST_STORAGE} as parent for temporary directories..."
 
 
 # 3. set up common vars and directories
 #    directory structure should be:
-#      ${EESSI_LOCAL_DISK}
+#      ${EESSI_HOST_STORAGE}
 #      |-singularity_cache
 #      |-${CVMFS_VAR_LIB}
 #      |-${CVMFS_VAR_RUN}
@@ -248,7 +240,7 @@ echo "Using ${EESSI_LOCAL_DISK} as parent for temporary directories..."
 source ${base_dir}/init/eessi_defaults
 
 # tmp dir for EESSI
-EESSI_TMPDIR=${EESSI_LOCAL_DISK}
+EESSI_TMPDIR=${EESSI_HOST_STORAGE}
 mkdir -p ${EESSI_TMPDIR}
 [[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}"
 
@@ -310,7 +302,7 @@ if [[ "${ACCESS}" == "rw" ]]; then
 fi
 
 
-# 5. initialize local disk/tmp from previous run if provided
+# 5. initialize host storage/tmp from previous run if provided
 
 
 # 6. run container
diff --git a/utils.sh b/utils.sh
index 0c98c86ec4..82dfd4a345 100644
--- a/utils.sh
+++ b/utils.sh
@@ -10,9 +10,14 @@ function echo_yellow() {
     echo -e "\e[33m$1\e[0m"
 }
 
+ANY_ERROR_EXITCODE=1
 function fatal_error() {
     echo_red "ERROR: $1" >&2
-    exit 1
+    if [[ $# -gt 1 ]]; then
+      exit $2
+    else
+      exit "${ANY_ERROR_EXITCODE}"
+    fi
 }
 
 function check_exit_code {
@@ -26,3 +31,61 @@ function check_exit_code {
         fatal_error "${fail_msg}"
     fi
 }
+
+declare -A repo_cfg
+
+function cfg_get_section {
+  if [[ "$1" =~ ^(\[)(.*)(\])$ ]]; then
+    echo ${BASH_REMATCH[2]}
+  else
+    echo ""
+  fi
+}
+
+function cfg_get_key_value {
+  if [[ "$1" =~ ^([^=]+)=([^=]+)$ ]]; then
+    echo "${BASH_REMATCH[1]}=${BASH_REMATCH[2]}"
+  else
+    echo ""
+  fi
+}
+
+function cfg_load {
+  local cur_section=""
+  local cur_key=""
+  local cur_val=""
+  IFS=
+  while read -r line; do
+    new_section=$(cfg_get_section $line)
+    # got a new section
+    if [[ -n "$new_section" ]]; then
+      cur_section=$new_section
+    # not a section, try a key value
+    else
+      val=$(cfg_get_key_value $line)
+      # trim leading and trailing spaces as well
+      cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//')
+      cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//')
+      if [[ -n "$cur_key" ]]; then
+        # section + key is the associative in bash array, the field separator is space
+        repo_cfg[${cur_section} ${cur_key}]=$cur_val
+      fi
+    fi
+  done <$1
+}
+
+function cfg_print {
+    for i in "${!repo_cfg[@]}"
+    do
+    # split the associative key in to section and key
+       echo -n "section  : $(echo $i | cut -f1 -d ' ');"
+       echo -n "key  : $(echo $i | cut -f2 -d ' ');"
+       echo  "value: ${repo_cfg[$i]}"
+    done
+}
+
+function cfg_get_value {
+    section=$1
+    key=$2
+    echo "${repo_cfg[$section $key]}"
+}

From 84daaa36c1e06baf3cd0de9f93ac9cf26c354cd3 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Tue, 3 Jan 2023 14:43:15 +0100
Subject: [PATCH 014/157] moving cfg handling to separate file cfg_files.sh

---
 cfg_files.sh | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 utils.sh     | 58 -----------------------------------------
 2 files changed, 73 insertions(+), 58 deletions(-)
 create mode 100644 cfg_files.sh

diff --git a/cfg_files.sh b/cfg_files.sh
new file mode 100644
index 0000000000..c2c549d0bf
--- /dev/null
+++ b/cfg_files.sh
@@ -0,0 +1,73 @@
+# functions for working with ini/cfg files
+#
+# This file is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Thomas Roeblitz (@trz42)
+#
+# license: GPLv2
+#
+
+
+# global variables
+# -a -> indexed array
+# -A -> associative array
+declare -A cfg_repos
+
+
+# functions
+function cfg_get_section {
+  if [[ "$1" =~ ^(\[)(.*)(\])$ ]]; then
+    echo ${BASH_REMATCH[2]}
+  else
+    echo ""
+  fi
+}
+
+function cfg_get_key_value {
+  if [[ "$1" =~ ^([^=]+)=([^=]+)$ ]]; then
+    echo "${BASH_REMATCH[1]}=${BASH_REMATCH[2]}"
+  else
+    echo ""
+  fi
+}
+
+function cfg_load {
+  local cur_section=""
+  local cur_key=""
+  local cur_val=""
+  IFS=
+  while read -r line; do
+    new_section=$(cfg_get_section $line)
+    # got a new section
+    if [[ -n "$new_section" ]]; then
+      cur_section=$new_section
+    # not a section, try a key value
+    else
+      val=$(cfg_get_key_value $line)
+      # trim leading and trailing spaces as well
+      cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//')
+      cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//')
+      if [[ -n "$cur_key" ]]; then
+        # section + key is the associative in bash array, the field separator is space
+        repo_cfg[${cur_section} ${cur_key}]=$cur_val
+      fi
+    fi
+  done <$1
+}
+
+function cfg_print {
+    for i in "${!repo_cfg[@]}"
+    do
+    # split the associative key in to section and key
+       echo -n "section  : $(echo $i | cut -f1 -d ' ');"
+       echo -n "key  : $(echo $i | cut -f2 -d ' ');"
+       echo  "value: ${repo_cfg[$i]}"
+    done
+}
+
+function cfg_get_value {
+    section=$1
+    key=$2
+    echo "${repo_cfg[$section $key]}"
+}
diff --git a/utils.sh b/utils.sh
index 82dfd4a345..5d8455bb68 100644
--- a/utils.sh
+++ b/utils.sh
@@ -31,61 +31,3 @@ function check_exit_code {
         fatal_error "${fail_msg}"
     fi
 }
-
-declare -A repo_cfg
-
-function cfg_get_section {
-  if [[ "$1" =~ ^(\[)(.*)(\])$ ]]; then
-    echo ${BASH_REMATCH[2]}
-  else
-    echo ""
-  fi
-}
-
-function cfg_get_key_value {
-  if [[ "$1" =~ ^([^=]+)=([^=]+)$ ]]; then
-    echo "${BASH_REMATCH[1]}=${BASH_REMATCH[2]}"
-  else
-    echo ""
-  fi
-}
-
-function cfg_load {
-  local cur_section=""
-  local cur_key=""
-  local cur_val=""
-  IFS=
-  while read -r line; do
-    new_section=$(cfg_get_section $line)
-    # got a new section
-    if [[ -n "$new_section" ]]; then
-      cur_section=$new_section
-    # not a section, try a key value
-    else
-      val=$(cfg_get_key_value $line)
-      # trim leading and trailing spaces as well
-      cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//')
-      cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//')
-      if [[ -n "$cur_key" ]]; then
-        # section + key is the associative in bash array, the field separator is space
-        repo_cfg[${cur_section} ${cur_key}]=$cur_val
-      fi
-    fi
-  done <$1
-}
-
-function cfg_print {
-    for i in "${!repo_cfg[@]}"
-    do
-    # split the associative key in to section and key
-       echo -n "section  : $(echo $i | cut -f1 -d ' ');"
-       echo -n "key  : $(echo $i | cut -f2 -d ' ');"
-       echo  "value: ${repo_cfg[$i]}"
-    done
-}
-
-function cfg_get_value {
-    section=$1
-    key=$2
-    echo "${repo_cfg[$section $key]}"
-}

From 95c65049ac6148fb531841131e431a585c540a16 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Tue, 3 Jan 2023 20:31:27 +0100
Subject: [PATCH 015/157] functions to process repository config file and use
 data in eessi_container.sh

---
 cfg_files.sh       | 87 ++++++++++++++++++++++++++++++++++++++++++----
 eessi_container.sh | 64 ++++++++++++++++++++++++++++++++--
 2 files changed, 142 insertions(+), 9 deletions(-)

diff --git a/cfg_files.sh b/cfg_files.sh
index c2c549d0bf..f17b893280 100644
--- a/cfg_files.sh
+++ b/cfg_files.sh
@@ -13,6 +13,7 @@
 # -a -> indexed array
 # -A -> associative array
 declare -A cfg_repos
+declare -A cfg_file_map
 
 
 # functions
@@ -46,28 +47,102 @@ function cfg_load {
     else
       val=$(cfg_get_key_value $line)
       # trim leading and trailing spaces as well
-      cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//')
-      cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//')
+      #cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//')
+      cur_key=$(echo $val | cut -f1 -d'=' | cfg_trim_spaces)
+      #cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//')
+      cur_val=$(echo $val | cut -f2 -d'=' | cfg_trim_spaces)
       if [[ -n "$cur_key" ]]; then
         # section + key is the associative in bash array, the field separator is space
-        repo_cfg[${cur_section} ${cur_key}]=$cur_val
+        cfg_repos[${cur_section} ${cur_key}]=$cur_val
       fi
     fi
   done <$1
 }
 
 function cfg_print {
-    for i in "${!repo_cfg[@]}"
+    for i in "${!cfg_repos[@]}"
     do
     # split the associative key in to section and key
        echo -n "section  : $(echo $i | cut -f1 -d ' ');"
        echo -n "key  : $(echo $i | cut -f2 -d ' ');"
-       echo  "value: ${repo_cfg[$i]}"
+       echo  "value: ${cfg_repos[$i]}"
     done
 }
 
 function cfg_get_value {
     section=$1
     key=$2
-    echo "${repo_cfg[$section $key]}"
+    echo "${cfg_repos[$section $key]}"
+}
+
+function cfg_trim_spaces {
+  # reads from argument $1 or stdin
+  if [[ $# -gt 0 ]]; then
+    sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' <<< ${1}
+  else
+    sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' < /dev/stdin
+  fi
+}
+
+function cfg_trim_quotes {
+  # reads from argument $1 or stdin
+  if [[ $# -gt 0 ]]; then
+    sed -e 's/^"*//' -e 's/"*$//' <<< ${1}
+  else
+    sed -e 's/^"*//' -e 's/"*$//' < /dev/stdin
+  fi
+}
+
+function cfg_trim_curly_brackets {
+  # reads from argument $1 or stdin
+  if [[ $# -gt 0 ]]; then
+    sed -e 's/^{*//' -e 's/}*$//' <<< ${1}
+  else
+    sed -e 's/^{*//' -e 's/}*$//' < /dev/stdin
+  fi
+}
+
+function cfg_get_all_sections {
+  # first field in keys
+  # 1. get first field in all keys, 2. filter duplicates, 3. return them as string
+  declare -A all_sections
+  for key in "${!cfg_repos[@]}"
+  do
+    section=$(echo "$key" | cut -f1 -d' ')
+    all_sections[${section}]=1
+  done
+  sections=
+  for sec_key in "${!all_sections[@]}"
+  do
+    sections="${sections} ${sec_key}"
+  done
+  echo "${sections}"
+}
+
+function cfg_init_file_map {
+  # strip '{' and '}' from config_map
+  # split config_map at ','
+  # for each item: split at ':' use first as key, second as value
+
+  # reset global variable
+  cfg_file_map=()
+
+  # expects a string containing the config_map from the cfg file
+  # trim leading and trailing curly brackets
+  cm_trimmed=$(cfg_trim_curly_brackets $1)
+
+  # split into elements along ','
+  declare -a cm_mappings
+#  IFS=',' read -r -a cm_mappings <<< "${cm_trimmed}"
+
+  for index in "${!cm_mappings[@]}"
+  do
+    # split mapping into key and value
+    map_key=$(echo ${cm_mappings[index]} | cut -f1 -d':')
+    map_value=$(echo ${cm_mappings[index]} | cut -f2 -d':')
+    # trim spaces and double quotes at start and end
+    tr_key=$(cfg_trim_spaces "${map_key}" | cfg_trim_quotes)
+    tr_value=$(cfg_trim_spaces "${map_value}" | cfg_trim_quotes)
+    cfg_file_map[${tr_key}]=${tr_value}
+  done
 }
diff --git a/eessi_container.sh b/eessi_container.sh
index b597b17647..be0bcccab6 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -26,6 +26,7 @@
 base_dir=$(dirname $(realpath $0))
 
 source ${base_dir}/utils.sh
+source ${base_dir}/cfg_files.sh
 
 # exit codes: bitwise shift codes to allow for combination of exit codes
 # ANY_ERROR_EXITCODE is sourced from ${base_dir}/utils.sh
@@ -44,6 +45,9 @@ RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11))
 CVMFS_VAR_LIB="var-lib-cvmfs"
 CVMFS_VAR_RUN="var-run-cvmfs"
 
+# repository cfg file
+REPO_CFG_FILE=repos.cfg
+
 
 # 0. parse args
 #    see example parsing of command line arguments at
@@ -131,6 +135,10 @@ while [[ $# -gt 0 ]]; do
       PREVIOUS_RUN="$2"
       shift 2
       ;;
+    -r|--repository)
+      REPOSITORY="$2"
+      shift 2
+      ;;
     -x|--http-proxy)
       HTTP_PROXY="$2"
       export http_proxy=${HTTP_PROXY}
@@ -236,6 +244,7 @@ echo "Using ${EESSI_HOST_STORAGE} as parent for temporary directories..."
 #      |-overlay-upper
 #      |-overlay-work
 #      |-home
+#      |-cfg
 
 source ${base_dir}/init/eessi_defaults
 
@@ -269,11 +278,60 @@ BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/ru
 BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp"
 [[ ${INFO} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}"
 
+# set up repository config (always create cfg dir and populate it with info when
+# arg -r|--repository is used)
+mkdir -p ${EESSI_TMPDIR}/cfg
+if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then
+  # strip "/cvmfs/" from default setting
+  repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//}
+else
+  # TODO implement more flexible specification of repo cfgs
+  #      REPOSITORY => repo-id OR repo-cfg-file (with a single section) OR
+  #                    repo-cfg-file:repo-id (repo-id defined in repo-cfg-file)
+  #
+  # for now, assuming repo-id is defined in config file pointed to
+  #   REPO_CFG_FILE, which is to be copied into the working directory
+  #   (could also become part of the software layer to define multiple
+  #    standard EESSI repositories)
+  cfg_load ${REPO_CFG_FILE}
+
+  # cfg file should include: repo_name, repo_version, config_bundle,
+  #   map { local_filepath -> container_filepath }
+  #
+  # repo_name_domain is the domain part of the repo_name, e.g.,
+  #   eessi-hpc.org for pilot.eessi-hpc.org
+  #
+  # where config bundle includes the files (-> target location in container)
+  # - default.local -> /etc/cvmfs/default.local
+  #   contains CVMFS settings, e.g., CVMFS_HTTP_PROXY, CVMFS_QUOTA_LIMIT, ...
+  # - ${repo_name_domain}.conf -> /etc/cvmfs/domain.d/${repo_name_domain}.conf
+  #   contains CVMFS settings, e.g., CVMFS_SERVER_URL (Stratum 1s),
+  #   CVMFS_KEYS_DIR, CVMFS_USE_GEOAPI, ...
+  # - ${repo_name_domain}/ -> /etc/cvmfs/keys/${repo_name_domain}
+  #   a directory that contains the public key to access the repository, key
+  #   itself then doesn't need to be BIND mounted
+  # - ${repo_name_domain}/${repo_name}.pub
+  #   (-> /etc/cvmfs/keys/${repo_name_domain}/${repo_name}.pub
+  #   the public key to access the repository, key itself is BIND mounted
+  #   via directory ${repo_name_domain}
+  repo_name=$(cfg_get_value ${REPOSITORY} "repo_name")
+  # derive domain part from repo_name (everything after first '.')
+  repo_name_domain=${repo_name#*.}
+  repo_version=$(cfg_get_value ${REPOSITORY} "repo_version")
+  config_bundle=$(cfg_get_value ${REPOSITORY} "config_bundle")
+  config_map=$(cfg_get_value ${REPOSITORY} "config_map")
+
+  # convert config_map into associative array cfg_file_map
+  cfg_init_file_map
+  declare -p cfg_file_map
+
+  # TODO use information to set up dir ${EESSI_TMPDIR}/cfg and
+  #      define BIND mounts
+  exit -1
+fi
 
-# 4. set up vars and dirs specific to a scenario
 
-# strip "/cvmfs/" from default setting
-repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//}
+# 4. set up vars and dirs specific to a scenario
 
 declare -a EESSI_FUSE_MOUNTS=()
 if [[ "${ACCESS}" == "ro" ]]; then

From f349b1ac841e82ea8c1b739ce5c883cc75c308d3 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Tue, 3 Jan 2023 20:51:59 +0100
Subject: [PATCH 016/157] small fixes and code style improvements

---
 cfg_files.sh       | 30 +++++++++++++++++++-----------
 eessi_container.sh |  4 ++--
 2 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/cfg_files.sh b/cfg_files.sh
index f17b893280..e1694bfc79 100644
--- a/cfg_files.sh
+++ b/cfg_files.sh
@@ -60,19 +60,19 @@ function cfg_load {
 }
 
 function cfg_print {
-    for i in "${!cfg_repos[@]}"
-    do
+  for index in "${!cfg_repos[@]}"
+  do
     # split the associative key in to section and key
-       echo -n "section  : $(echo $i | cut -f1 -d ' ');"
-       echo -n "key  : $(echo $i | cut -f2 -d ' ');"
-       echo  "value: ${cfg_repos[$i]}"
-    done
+    echo -n "section  : $(echo $index | cut -f1 -d ' ');"
+    echo -n "key  : $(echo $index | cut -f2 -d ' ');"
+    echo  "value: ${cfg_repos[$index]}"
+  done
 }
 
 function cfg_get_value {
-    section=$1
-    key=$2
-    echo "${cfg_repos[$section $key]}"
+  section=$1
+  key=$2
+  echo "${cfg_repos[$section $key]}"
 }
 
 function cfg_trim_spaces {
@@ -116,7 +116,7 @@ function cfg_get_all_sections {
   do
     sections="${sections} ${sec_key}"
   done
-  echo "${sections}"
+  echo "${sections}" | cfg_trim_spaces
 }
 
 function cfg_init_file_map {
@@ -129,7 +129,7 @@ function cfg_init_file_map {
 
   # expects a string containing the config_map from the cfg file
   # trim leading and trailing curly brackets
-  cm_trimmed=$(cfg_trim_curly_brackets $1)
+  cm_trimmed=$(cfg_trim_curly_brackets "$1")
 
   # split into elements along ','
   declare -a cm_mappings
@@ -146,3 +146,11 @@ function cfg_init_file_map {
     cfg_file_map[${tr_key}]=${tr_value}
   done
 }
+
+function cfg_print_map {
+  for index in "${!cfg_file_map[@]}"
+  do
+    echo "${index} --> ${cfg_file_map[${index}]}"
+  done
+}
+
diff --git a/eessi_container.sh b/eessi_container.sh
index be0bcccab6..85f1f55077 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -322,8 +322,8 @@ else
   config_map=$(cfg_get_value ${REPOSITORY} "config_map")
 
   # convert config_map into associative array cfg_file_map
-  cfg_init_file_map
-  declare -p cfg_file_map
+  cfg_init_file_map "${config_map}"
+  cfg_print_map
 
   # TODO use information to set up dir ${EESSI_TMPDIR}/cfg and
   #      define BIND mounts

From f1a583c22912c0b1a5bc9c218f3a75a6c5246322 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Tue, 3 Jan 2023 21:01:38 +0100
Subject: [PATCH 017/157] uncommented reading of config map

---
 cfg_files.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cfg_files.sh b/cfg_files.sh
index e1694bfc79..885ebd0877 100644
--- a/cfg_files.sh
+++ b/cfg_files.sh
@@ -133,7 +133,7 @@ function cfg_init_file_map {
 
   # split into elements along ','
   declare -a cm_mappings
-#  IFS=',' read -r -a cm_mappings <<< "${cm_trimmed}"
+  IFS=',' read -r -a cm_mappings <<< "${cm_trimmed}"
 
   for index in "${!cm_mappings[@]}"
   do

From ff207f3ad866f3b7c815333f8c964a952f6511b2 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Wed, 4 Jan 2023 10:33:55 +0100
Subject: [PATCH 018/157] preparing config information for custom repository

---
 eessi_container.sh | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 85f1f55077..1e1ed8e4ce 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -36,7 +36,7 @@ CONTAINER_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 3))
 HOST_STORAGE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 4))
 MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5))
 PREVIOUS_RUN_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6))
-REPOSITORY_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7))
+REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7))
 HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9))
 HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10))
 RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11))
@@ -185,7 +185,7 @@ fi
 # PREVIOUS_RUN_ERROR_EXITCODE
 
 # TODO (arg -r|--repository) check if repository is known
-# REPOSITORY_UNKNOWN_EXITCODE
+# REPOSITORY_ERROR_EXITCODE
 
 # TODO (arg -x|--http-proxy) check if http proxy is accessible
 # HTTP_PROXY_ERROR_EXITCODE
@@ -246,8 +246,6 @@ echo "Using ${EESSI_HOST_STORAGE} as parent for temporary directories..."
 #      |-home
 #      |-cfg
 
-source ${base_dir}/init/eessi_defaults
-
 # tmp dir for EESSI
 EESSI_TMPDIR=${EESSI_HOST_STORAGE}
 mkdir -p ${EESSI_TMPDIR}
@@ -282,6 +280,9 @@ BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp"
 # arg -r|--repository is used)
 mkdir -p ${EESSI_TMPDIR}/cfg
 if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then
+  # need to source defaults as late as possible (see other sourcing below)
+  source ${base_dir}/init/eessi_defaults
+
   # strip "/cvmfs/" from default setting
   repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//}
 else
@@ -325,9 +326,22 @@ else
   cfg_init_file_map "${config_map}"
   cfg_print_map
 
-  # TODO use information to set up dir ${EESSI_TMPDIR}/cfg and
-  #      define BIND mounts
-  exit -1
+  # TODO use information to set up dir ${EESSI_TMPDIR}/cfg,
+  #      define BIND mounts and override repo name and version
+  # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/cfg
+  if [[ ! -r ${config_bundle} ]]; then
+    fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE}
+  fi
+  tar xf ${config_bundle} -C ${EESSI_TMPDIR}/cfg
+  for src in "${!cfg_file_map[@]}"
+  do
+    target=${cfg_file_map[${src}]}
+    BIND_PATHS="${BIND_PATHS},${src}:${target}"
+  done
+  export EESSI_PILOT_VERSION_OVERRIDE=${repo_version}
+  export EESSI_CVMFS_REPO_OVERRIDE=${repo_name}
+  # need to source defaults as late as possible (after *_OVERRIDEs)
+  source ${base_dir}/init/eessi_defaults
 fi
 
 

From b2543b9f70a27ad0be01159797d408362c718836 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Wed, 4 Jan 2023 10:35:42 +0100
Subject: [PATCH 019/157] fix source location for config files

---
 eessi_container.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 1e1ed8e4ce..cc8ab956da 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -336,7 +336,7 @@ else
   for src in "${!cfg_file_map[@]}"
   do
     target=${cfg_file_map[${src}]}
-    BIND_PATHS="${BIND_PATHS},${src}:${target}"
+    BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/cfg/${src}:${target}"
   done
   export EESSI_PILOT_VERSION_OVERRIDE=${repo_version}
   export EESSI_CVMFS_REPO_OVERRIDE=${repo_name}

From 3f161976d2ff1567e6a197bf91562f8d4aa122e6 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Wed, 4 Jan 2023 14:37:32 +0100
Subject: [PATCH 020/157] using correct repository paths

---
 eessi_container.sh | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index cc8ab956da..1df3574a6a 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -339,7 +339,7 @@ else
     BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/cfg/${src}:${target}"
   done
   export EESSI_PILOT_VERSION_OVERRIDE=${repo_version}
-  export EESSI_CVMFS_REPO_OVERRIDE=${repo_name}
+  export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}"
   # need to source defaults as late as possible (after *_OVERRIDEs)
   source ${base_dir}/init/eessi_defaults
 fi
@@ -349,8 +349,10 @@ fi
 
 declare -a EESSI_FUSE_MOUNTS=()
 if [[ "${ACCESS}" == "ro" ]]; then
-  export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} ${EESSI_CVMFS_REPO}"
+  export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs/${repo_name}"
+
   EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_READONLY}")
+  export EESSI_FUSE_MOUNTS
 fi
 
 if [[ "${ACCESS}" == "rw" ]]; then
@@ -362,7 +364,8 @@ if [[ "${ACCESS}" == "rw" ]]; then
   [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}"
 
   # set environment variables for fuse mounts in Singularity container
-  EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}"
+  export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}"
+
   EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_READONLY}")
 
   EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs"
@@ -370,7 +373,10 @@ if [[ "${ACCESS}" == "rw" ]]; then
   EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=/tmp/overlay-upper"
   EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=/tmp/overlay-work"
   EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}"
+  export EESSI_PILOT_WRITABLE_OVERLAY
+
   EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_WRITABLE_OVERLAY}")
+  export EESSI_FUSE_MOUNTS
 fi
 
 

From 935517be94c0ea8b4ca5107aa7251da34a5473f4 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Wed, 4 Jan 2023 19:29:51 +0100
Subject: [PATCH 021/157] make repository cfg file path configurable

---
 eessi_container.sh | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 1df3574a6a..b8867d2d78 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -45,8 +45,9 @@ RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11))
 CVMFS_VAR_LIB="var-lib-cvmfs"
 CVMFS_VAR_RUN="var-run-cvmfs"
 
-# repository cfg file
-REPO_CFG_FILE=repos.cfg
+# repository cfg file, default name (default location: $PWD)
+#   can be overwritten by setting env var EESSI_REPOS_CFG_FILE_OVERRIDE
+export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}"
 
 
 # 0. parse args
@@ -291,10 +292,10 @@ else
   #                    repo-cfg-file:repo-id (repo-id defined in repo-cfg-file)
   #
   # for now, assuming repo-id is defined in config file pointed to
-  #   REPO_CFG_FILE, which is to be copied into the working directory
+  #   EESSI_REPOS_CFG_FILE, which is to be copied into the working directory
   #   (could also become part of the software layer to define multiple
   #    standard EESSI repositories)
-  cfg_load ${REPO_CFG_FILE}
+  cfg_load ${EESSI_REPOS_CFG_FILE}
 
   # cfg file should include: repo_name, repo_version, config_bundle,
   #   map { local_filepath -> container_filepath }

From d28cc696374cb0523a8583a4dcce58899628778b Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Wed, 4 Jan 2023 20:51:54 +0100
Subject: [PATCH 022/157] short section about eessi_container.sh

---
 README.md | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/README.md b/README.md
index 4d463b1c5b..daf02eebc2 100644
--- a/README.md
+++ b/README.md
@@ -19,6 +19,22 @@ Environment set up to use EESSI pilot software stack, have fun!
 [EESSI pilot 2021.12] $
 ```
 
+### Accessing EESSI via a container
+
+You need Singularity version 3.7 or newer. Then, simply run
+
+```
+$ ./eessi_container.sh
+```
+Once you get presented the prompt `Singularity>` run the above `source` command.
+
+If you want to build a package for the software repository, simply add the arguments `--access rw`, e.g., full command would be
+
+```
+$ ./eessi_container.sh --access rw
+```
+Note, not all features/arguments listed via `./eessi_container.sh --help` are implemented.
+
 # License
 
 The software in this repository is distributed under the terms of the

From eca4f9798213ddc275b7496c1923fd8915af906b Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Wed, 4 Jan 2023 20:54:31 +0100
Subject: [PATCH 023/157] show which features are not yet implemented

---
 eessi_container.sh | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index b8867d2d78..fafa54193b 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -61,14 +61,20 @@ display_help() {
   echo "  -a | --access {ro,rw}    -  ro (read-only), rw (read & write) [default: ro]"
   echo "  -c | --container IMAGE   -  image file or URL defining the container to use"
   echo "                           [default: docker://ghcr.io/eessi/build-node:debian10]"
-  echo "  -d | --dry-run           -  run script except for executing the container,"
-  echo "                              print information about setup [default: false]"
   echo "  -h | --help              -  display this usage information [default: false]"
-  echo "  -i | --info              -  display configured repositories [default: false]"
   echo "  -l | --host-storage DIR  -  directory space on host machine (used for"
   echo "                              temporary data) [default: 1. TMPDIR, 2. /tmp]"
   echo "  -m | --mode {shell,run}  -  shell (launch interactive shell)"
   echo "                              run (run a script) [default: shell]"
+  echo "  -r | --repository CFG    -  configuration file or identifier defining the"
+  echo "                              repository to use [default: EESSI-pilot]"
+  echo
+  echo " If value for --mode is 'run', the SCRIPT provided is executed."
+  echo
+  echo " FEATURES/OPTIONS to be implemented:"
+  echo "  -d | --dry-run           -  run script except for executing the container,"
+  echo "                              print information about setup [default: false]"
+  echo "  -i | --info              -  display configured repositories [default: false]"
   echo "  -p | --previous-run PRUN -  init local disk with data from previous run"
   echo "                              format is PATH[:TAR/ZIP] where PATH is pointing"
   echo "                              to the previously used local disk, and TAR/ZIP"
@@ -76,14 +82,10 @@ display_help() {
   echo "                              doesn't exist currently; if PATH exists and"
   echo "                              a TAR/ZIP is provided an error is reported"
   echo "                              [default: not set]"
-  echo "  -r | --repository CFG    -  configuration file or identifier defining the"
-  echo "                              repository to use [default: EESSI-pilot]"
   echo "  -x | --http-proxy URL    -  provides URL for the env variable http_proxy"
   echo "                              [default: not set]"
   echo "  -y | --https-proxy URL   -  provides URL for the env variable https_proxy"
   echo "                              [default: not set]"
-  echo
-  echo " If value for --mode is 'run', the SCRIPT provided is executed."
 }
 
 # set defaults for command line arguments

From 576897821c2eca3c21f7f168804315dec186466e Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Thu, 5 Jan 2023 19:53:55 +0100
Subject: [PATCH 024/157] adding support for resuming a previous session

---
 eessi_container.sh | 146 +++++++++++++++++++++++++--------------------
 1 file changed, 82 insertions(+), 64 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index fafa54193b..9048b1ae7d 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -35,7 +35,7 @@ ACCESS_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 2))
 CONTAINER_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 3))
 HOST_STORAGE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 4))
 MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5))
-PREVIOUS_RUN_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6))
+RESUME_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6))
 REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7))
 HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9))
 HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10))
@@ -58,16 +58,22 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}"
 display_help() {
   echo "usage: $0 [OPTIONS] [SCRIPT]"
   echo " OPTIONS:"
-  echo "  -a | --access {ro,rw}    -  ro (read-only), rw (read & write) [default: ro]"
-  echo "  -c | --container IMAGE   -  image file or URL defining the container to use"
-  echo "                           [default: docker://ghcr.io/eessi/build-node:debian10]"
-  echo "  -h | --help              -  display this usage information [default: false]"
-  echo "  -l | --host-storage DIR  -  directory space on host machine (used for"
-  echo "                              temporary data) [default: 1. TMPDIR, 2. /tmp]"
-  echo "  -m | --mode {shell,run}  -  shell (launch interactive shell)"
-  echo "                              run (run a script) [default: shell]"
-  echo "  -r | --repository CFG    -  configuration file or identifier defining the"
-  echo "                              repository to use [default: EESSI-pilot]"
+  echo "  -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]"
+  echo "  -c | --container IMG  - image file or URL defining the container to use"
+  echo "                          [default: docker://ghcr.io/eessi/build-node:debian10]"
+  echo "  -h | --help           - display this usage information [default: false]"
+  echo "  -g | --storage DIR    - directory space on host machine (used for"
+  echo "                          temporary data) [default: 1. TMPDIR, 2. /tmp]"
+  echo "  -m | --mode MODE      - with MODE==shell (launch interactive shell) or"
+  echo "                          MODE==run (run a script) [default: shell]"
+  echo "  -r | --repository CFG - configuration file or identifier defining the"
+  echo "                          repository to use [default: EESSI-pilot]"
+  echo "  -u | --resume DIR/TGZ - resume a previous run from a directory or tarball,"
+  echo "                          where DIR points to a previously used tmp directory"
+  echo "                          (check for output 'Using DIR as tmp' of previous run)"
+  echo "                          and TGZ is the path to a tarball which is unpacked"
+  echo "                          the tmp dir stored on the local storage space (see"
+  echo "                          option --storage above) [default: not set]"
   echo
   echo " If value for --mode is 'run', the SCRIPT provided is executed."
   echo
@@ -75,13 +81,6 @@ display_help() {
   echo "  -d | --dry-run           -  run script except for executing the container,"
   echo "                              print information about setup [default: false]"
   echo "  -i | --info              -  display configured repositories [default: false]"
-  echo "  -p | --previous-run PRUN -  init local disk with data from previous run"
-  echo "                              format is PATH[:TAR/ZIP] where PATH is pointing"
-  echo "                              to the previously used local disk, and TAR/ZIP"
-  echo "                              is used to initialize the local disk if PATH"
-  echo "                              doesn't exist currently; if PATH exists and"
-  echo "                              a TAR/ZIP is provided an error is reported"
-  echo "                              [default: not set]"
   echo "  -x | --http-proxy URL    -  provides URL for the env variable http_proxy"
   echo "                              [default: not set]"
   echo "  -y | --https-proxy URL   -  provides URL for the env variable https_proxy"
@@ -93,10 +92,10 @@ ACCESS="ro"
 CONTAINER="docker://ghcr.io/eessi/build-node:debian10"
 DRY_RUN=0
 INFO=0
-HOST_STORAGE=
+STORAGE=
 MODE="shell"
-PREVIOUS_RUN=
 REPOSITORY="EESSI-pilot"
+RESUME=
 HTTP_PROXY=
 HTTPS_PROXY=
 RUN_SCRIPT_AND_ARGS=
@@ -117,6 +116,10 @@ while [[ $# -gt 0 ]]; do
       DRY_RUN=1
       shift 1
       ;;
+    -g|--storage)
+      STORAGE="$2"
+      shift 2
+      ;;
     -h|--help)
       display_help
       exit 0
@@ -125,23 +128,18 @@ while [[ $# -gt 0 ]]; do
       INFO=1
       shift 1
       ;;
-    -l|--host-storage)
-      HOST_STORAGE="$2"
-      #EESSI_TMPDIR="$2"
-      shift 2
-      ;;
     -m|--mode)
       MODE="$2"
       shift 2
       ;;
-    -p|--previous-run)
-      PREVIOUS_RUN="$2"
-      shift 2
-      ;;
     -r|--repository)
       REPOSITORY="$2"
       shift 2
       ;;
+    -u|--resume)
+      RESUME="$2"
+      shift 2
+      ;;
     -x|--http-proxy)
       HTTP_PROXY="$2"
       export http_proxy=${HTTP_PROXY}
@@ -174,7 +172,7 @@ fi
 # TODO (arg -c|--container) check container (is it a file or URL & access those)
 # CONTAINER_ERROR_EXITCODE
 
-# TODO (arg -l|--host-storage) check if it exists, if user has write permission,
+# TODO (arg -g|--storage) check if it exists, if user has write permission,
 #      if it contains no data, etc.
 # HOST_STORAGE_ERROR_EXITCODE
 
@@ -183,13 +181,13 @@ if [[ "${MODE}" != "shell" && "${MODE}" != "run" ]]; then
   fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}"
 fi
 
-# TODO (arg -p|--previous-run) check if it exists, if user has read permission,
-#      if it contains data from a previous run
-# PREVIOUS_RUN_ERROR_EXITCODE
-
 # TODO (arg -r|--repository) check if repository is known
 # REPOSITORY_ERROR_EXITCODE
 
+# TODO (arg -u|--resume) check if it exists, if user has read permission,
+#      if it contains data from a previous run
+# RESUME_ERROR_EXITCODE
+
 # TODO (arg -x|--http-proxy) check if http proxy is accessible
 # HTTP_PROXY_ERROR_EXITCODE
 
@@ -206,37 +204,52 @@ if [[ "${MODE}" == "run" ]]; then
 fi
 
 
-# 2. set up host storage/tmp
-# as location for temporary data use in the following order
-#   a. command line argument -l|--host-storage
-#   b. env var TMPDIR
-#   c. /tmp
-# note, we ensure that (a) takes precedence by setting TMPDIR to HOST_STORAGE
-#     if HOST_STORAGE is not empty
-# note, (b) & (c) are automatically ensured by using 'mktemp -d --tmpdir' to
-#     create a temporary directory
-# note, if previous run is used the name of the temporary directory
-#     should be identical to previous run, ie, then we don't create a new
-#     temporary directory
-if [[ ! -z ${HOST_STORAGE} ]]; then
-  export TMPDIR=${HOST_STORAGE}
-  # mktemp fails if TMPDIR does not exist, so let's create it
-  mkdir -p ${TMPDIR}
-fi
-if [[ ! -z ${TMPDIR} ]]; then
-  # TODO check if TMPDIR already exists
-  # mktemp fails if TMPDIR does not exist, so let's create it
-  mkdir -p ${TMPDIR}
-fi
-if [[ -z ${TMPDIR} ]]; then
-  # mktemp falls back to using /tmp if TMPDIR is empty
-  # TODO check if /tmp is writable, large enough and usable (different
-  #      features for ro-access and rw-access)
-  echo "skipping sanity checks for /tmp"
+# 2. set up host storage/tmp if necessary
+# if session to be resumed from a previous one (--resume ARG) and ARG is a directory
+#   just reuse ARG, define environment variables accordingly and skip creating a new
+#   tmp storage
+if [[ ! -z ${RESUME} && -d ${RESUME} ]]; then
+  # resume from directory ${RESUME}
+  #   skip creating a new tmp directory, just set environment variables
+  echo "Resuming from previous run using temporary storage at ${RESUME}"
+  EESSI_HOST_STORAGE=${RESUME}
+else
+  # we need a tmp location (and possibly init it with ${RESUME} if it was not
+  #   a directory
+
+  # as location for temporary data use in the following order
+  #   a. command line argument -l|--host-storage
+  #   b. env var TMPDIR
+  #   c. /tmp
+  # note, we ensure that (a) takes precedence by setting TMPDIR to STORAGE
+  #     if STORAGE is not empty
+  # note, (b) & (c) are automatically ensured by using 'mktemp -d --tmpdir' to
+  #     create a temporary directory
+  if [[ ! -z ${STORAGE} ]]; then
+    export TMPDIR=${STORAGE}
+    # mktemp fails if TMPDIR does not exist, so let's create it
+    mkdir -p ${TMPDIR}
+  fi
+  if [[ ! -z ${TMPDIR} ]]; then
+    # TODO check if TMPDIR already exists
+    # mktemp fails if TMPDIR does not exist, so let's create it
+    mkdir -p ${TMPDIR}
+  fi
+  if [[ -z ${TMPDIR} ]]; then
+    # mktemp falls back to using /tmp if TMPDIR is empty
+    # TODO check if /tmp is writable, large enough and usable (different
+    #      features for ro-access and rw-access)
+    echo "skipping sanity checks for /tmp"
+  fi
+  EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX)
+  echo "Using ${EESSI_HOST_STORAGE} as parent for temporary directories..."
 fi
-EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX)
-echo "Using ${EESSI_HOST_STORAGE} as parent for temporary directories..."
 
+# if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE}
+if [[ ! -z ${RESUME} && -f ${RESUME} ]]; then
+  tar xf ${RESUME} -C ${EESSI_HOST_STORAGE}
+  echo "Resuming from previous run using temporary storage ${RESUME} unpacked into ${EESSI_HOST_STORAGE}"
+fi
 
 # 3. set up common vars and directories
 #    directory structure should be:
@@ -335,7 +348,12 @@ else
   if [[ ! -r ${config_bundle} ]]; then
     fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE}
   fi
-  tar xf ${config_bundle} -C ${EESSI_TMPDIR}/cfg
+
+  # only unpack config_bundle if we're not resuming from a previous run
+  if [[ -z ${RESUME} ]]; then
+    tar xf ${config_bundle} -C ${EESSI_TMPDIR}/cfg
+  fi
+
   for src in "${!cfg_file_map[@]}"
   do
     target=${cfg_file_map[${src}]}

From bf90a48478f810efbf12e227041069145b5423b9 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Thu, 5 Jan 2023 20:06:23 +0100
Subject: [PATCH 025/157] improving message for tmp and resume option

---
 eessi_container.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 9048b1ae7d..71efdd0c01 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -70,10 +70,10 @@ display_help() {
   echo "                          repository to use [default: EESSI-pilot]"
   echo "  -u | --resume DIR/TGZ - resume a previous run from a directory or tarball,"
   echo "                          where DIR points to a previously used tmp directory"
-  echo "                          (check for output 'Using DIR as tmp' of previous run)"
-  echo "                          and TGZ is the path to a tarball which is unpacked"
-  echo "                          the tmp dir stored on the local storage space (see"
-  echo "                          option --storage above) [default: not set]"
+  echo "                          (check for output 'Using DIR as tmp ...' of a previous"
+  echo "                          run) and TGZ is the path to a tarball which is"
+  echo "                          unpacked the tmp dir stored on the local storage space"
+  echo "                          (see option --storage above) [default: not set]"
   echo
   echo " If value for --mode is 'run', the SCRIPT provided is executed."
   echo
@@ -242,7 +242,7 @@ else
     echo "skipping sanity checks for /tmp"
   fi
   EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX)
-  echo "Using ${EESSI_HOST_STORAGE} as parent for temporary directories..."
+  echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)."
 fi
 
 # if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE}

From 6808ec587246a1d92f6fd06cf5e705218d57e001 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 6 Jan 2023 11:24:16 +0100
Subject: [PATCH 026/157] first attempt at implementing --save option

---
 eessi_container.sh | 43 +++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 41 insertions(+), 2 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 71efdd0c01..e19e44ca91 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -35,8 +35,9 @@ ACCESS_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 2))
 CONTAINER_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 3))
 HOST_STORAGE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 4))
 MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5))
-RESUME_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6))
-REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7))
+REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6))
+RESUME_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7))
+SAVE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 8))
 HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9))
 HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10))
 RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11))
@@ -74,6 +75,11 @@ display_help() {
   echo "                          run) and TGZ is the path to a tarball which is"
   echo "                          unpacked the tmp dir stored on the local storage space"
   echo "                          (see option --storage above) [default: not set]"
+  echo "  -s | --save DIR/TGZ   - save contents of tmp directory to a tarball in"
+  echo "                          directory DIR or provided with the fixed full path TGZ"
+  echo "                          when a directory is provided, the format of the"
+  echo "                          tarball's name will be"
+  echo "                          {REPO_ID}-{OS}-{ARCH}-{TIMESTAMP}.tgz [default: not set]"
   echo
   echo " If value for --mode is 'run', the SCRIPT provided is executed."
   echo
@@ -96,6 +102,7 @@ STORAGE=
 MODE="shell"
 REPOSITORY="EESSI-pilot"
 RESUME=
+SAVE=
 HTTP_PROXY=
 HTTPS_PROXY=
 RUN_SCRIPT_AND_ARGS=
@@ -136,6 +143,10 @@ while [[ $# -gt 0 ]]; do
       REPOSITORY="$2"
       shift 2
       ;;
+    -s|--save)
+      SAVE="$2"
+      shift 2
+      ;;
     -u|--resume)
       RESUME="$2"
       shift 2
@@ -188,6 +199,11 @@ fi
 #      if it contains data from a previous run
 # RESUME_ERROR_EXITCODE
 
+# TODO (arg -s|--save) check if DIR exists, if user has write permission,
+#   if TGZ already exists, if user has write permission to directory to which
+#   TGZ should be written
+# SAVE_ERROR_EXITCODE
+
 # TODO (arg -x|--http-proxy) check if http proxy is accessible
 # HTTP_PROXY_ERROR_EXITCODE
 
@@ -416,3 +432,26 @@ fi
 echo "Launching container with command (next line):"
 echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}"
 singularity ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}
+
+# 7. save tmp if requested (arg -s|--save)
+if [[ ! -z ${SAVE} ]]; then
+  if [[ -d ${SAVE} ]]; then
+    # TODO to be implemented; best way a bit unclear
+    # assume SAVE is name of a directory to which tarball shall be written to
+    #   name format: {REPO_ID}-{OS}-{ARCH}-{TIMESTAMP}.tgz
+    # use container to determine OS and ARCH (same approach as in eessi-bot-build.slurm)
+    #RUN_CMD="python3 /mnt/os_arch_detect/eessi_software_subdir.py ${GENERIC_OPT}"
+    # only need READONLY part for CVMFS
+    #singularity run -B ... ${CONTAINER} ${RUN_CMD} 2>&1)
+    os=foo
+    arch=bar
+    ts=$(date +%s)
+    TGZ=${REPOSITORY}-${os}-${arch}-${ts}.tgz
+  else
+    # assume SAVE is the full path to a tarball's name
+    tar cf ${SAVE} -C ${EESSI_TMPDIR} .
+    echo "Save contents of '${EESSI_TMPDIR}' to '${SAVE}' (to resume, add '--resume ${SAVE}')"
+  fi
+fi
+
+# TODO clean up tmp by default? only retain if another option provided (--retain-tmp)

From efe93c72c6e782bc08535cb9045ee1474c3e98e2 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 6 Jan 2023 11:57:05 +0100
Subject: [PATCH 027/157] added support for providing just a directory to
 deposit a tarball of tmp

---
 eessi_container.sh | 24 +++++++++++-------------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index e19e44ca91..01638ebf03 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -78,8 +78,8 @@ display_help() {
   echo "  -s | --save DIR/TGZ   - save contents of tmp directory to a tarball in"
   echo "                          directory DIR or provided with the fixed full path TGZ"
   echo "                          when a directory is provided, the format of the"
-  echo "                          tarball's name will be"
-  echo "                          {REPO_ID}-{OS}-{ARCH}-{TIMESTAMP}.tgz [default: not set]"
+  echo "                          tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz"
+  echo "                          [default: not set]"
   echo
   echo " If value for --mode is 'run', the SCRIPT provided is executed."
   echo
@@ -435,23 +435,21 @@ singularity ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS
 
 # 7. save tmp if requested (arg -s|--save)
 if [[ ! -z ${SAVE} ]]; then
+  # Note, for now we don't try to be smart and record in any way the OS and
+  #   ARCH which might have been used internally, eg, when software packages
+  #   were built ... we rather keep the script here "stupid" and leave the handling
+  #   of these aspects to where the script is used
   if [[ -d ${SAVE} ]]; then
-    # TODO to be implemented; best way a bit unclear
     # assume SAVE is name of a directory to which tarball shall be written to
-    #   name format: {REPO_ID}-{OS}-{ARCH}-{TIMESTAMP}.tgz
-    # use container to determine OS and ARCH (same approach as in eessi-bot-build.slurm)
-    #RUN_CMD="python3 /mnt/os_arch_detect/eessi_software_subdir.py ${GENERIC_OPT}"
-    # only need READONLY part for CVMFS
-    #singularity run -B ... ${CONTAINER} ${RUN_CMD} 2>&1)
-    os=foo
-    arch=bar
+    #   name format: {REPO_ID}-{TIMESTAMP}.tgz
     ts=$(date +%s)
-    TGZ=${REPOSITORY}-${os}-${arch}-${ts}.tgz
+    TGZ=${SAVE}/${REPOSITORY}-${ts}.tgz
   else
     # assume SAVE is the full path to a tarball's name
-    tar cf ${SAVE} -C ${EESSI_TMPDIR} .
-    echo "Save contents of '${EESSI_TMPDIR}' to '${SAVE}' (to resume, add '--resume ${SAVE}')"
+    TGZ=${SAVE}
   fi
+  tar cf ${TGZ} -C ${EESSI_TMPDIR} .
+  echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')"
 fi
 
 # TODO clean up tmp by default? only retain if another option provided (--retain-tmp)

From f896c51b73272b7b9f3dab6a12b6a700292e0a12 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 6 Jan 2023 12:15:07 +0100
Subject: [PATCH 028/157] cleaning up code, making script more silent by
 default

---
 eessi_container.sh | 44 +++++++++++++++++++++++---------------------
 1 file changed, 23 insertions(+), 21 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 01638ebf03..57b003368b 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -38,8 +38,8 @@ MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5))
 REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6))
 RESUME_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7))
 SAVE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 8))
-HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9))
-HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10))
+#HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9))
+#HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10))
 RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11))
 
 # CernVM-FS settings
@@ -87,10 +87,10 @@ display_help() {
   echo "  -d | --dry-run           -  run script except for executing the container,"
   echo "                              print information about setup [default: false]"
   echo "  -i | --info              -  display configured repositories [default: false]"
-  echo "  -x | --http-proxy URL    -  provides URL for the env variable http_proxy"
-  echo "                              [default: not set]"
-  echo "  -y | --https-proxy URL   -  provides URL for the env variable https_proxy"
-  echo "                              [default: not set]"
+  #echo "  -x | --http-proxy URL    -  provides URL for the env variable http_proxy"
+  #echo "                              [default: not set]"
+  #echo "  -y | --https-proxy URL   -  provides URL for the env variable https_proxy"
+  #echo "                              [default: not set]"
 }
 
 # set defaults for command line arguments
@@ -103,8 +103,8 @@ MODE="shell"
 REPOSITORY="EESSI-pilot"
 RESUME=
 SAVE=
-HTTP_PROXY=
-HTTPS_PROXY=
+#HTTP_PROXY=
+#HTTPS_PROXY=
 RUN_SCRIPT_AND_ARGS=
 
 POSITIONAL_ARGS=()
@@ -151,16 +151,16 @@ while [[ $# -gt 0 ]]; do
       RESUME="$2"
       shift 2
       ;;
-    -x|--http-proxy)
-      HTTP_PROXY="$2"
-      export http_proxy=${HTTP_PROXY}
-      shift 2
-      ;;
-    -y|--https-proxy)
-      HTTPS_PROXY="$2"
-      export https_proxy=${HTTPS_PROXY}
-      shift 2
-      ;;
+#    -x|--http-proxy)
+#      HTTP_PROXY="$2"
+#      export http_proxy=${HTTP_PROXY}
+#      shift 2
+#      ;;
+#    -y|--https-proxy)
+#      HTTPS_PROXY="$2"
+#      export https_proxy=${HTTPS_PROXY}
+#      shift 2
+#      ;;
     -*|--*)
       fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}"
       ;;
@@ -255,7 +255,7 @@ else
     # mktemp falls back to using /tmp if TMPDIR is empty
     # TODO check if /tmp is writable, large enough and usable (different
     #      features for ro-access and rw-access)
-    echo "skipping sanity checks for /tmp"
+    [[ ${INFO} -eq 1 ]] && echo "skipping sanity checks for /tmp"
   fi
   EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX)
   echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)."
@@ -356,7 +356,7 @@ else
 
   # convert config_map into associative array cfg_file_map
   cfg_init_file_map "${config_map}"
-  cfg_print_map
+  [[ ${INFO} -eq 1 ]] && cfg_print_map
 
   # TODO use information to set up dir ${EESSI_TMPDIR}/cfg,
   #      define BIND mounts and override repo name and version
@@ -431,7 +431,9 @@ fi
 
 echo "Launching container with command (next line):"
 echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}"
-singularity ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}
+# TODO for now we run singularity with '-q' (quiet), later adjust this to the log level
+#      provided to the script
+singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}
 
 # 7. save tmp if requested (arg -s|--save)
 if [[ ! -z ${SAVE} ]]; then

From 94ca9126a233fc1109adf3c815d6c62277b8bcb3 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 15 Jan 2023 20:59:15 +0100
Subject: [PATCH 029/157] Fix hash for checkout action in README CI

---
 .github/workflows/tests_readme.yml | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/tests_readme.yml b/.github/workflows/tests_readme.yml
index 29dedcc3ff..5c6d0318d4 100644
--- a/.github/workflows/tests_readme.yml
+++ b/.github/workflows/tests_readme.yml
@@ -18,14 +18,15 @@ jobs:
   build:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v2
+        - name: Check out software-layer repository
+          uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
-    - name: verify if README.md is consistent with EESSI_PILOT_VERSION from init/eessi_defaults
-      run: |
-        source init/eessi_defaults
-        grep "${EESSI_PILOT_VERSION}" README.md
+        - name: verify if README.md is consistent with EESSI_PILOT_VERSION from init/eessi_defaults
+          run: |
+            source init/eessi_defaults
+            grep "${EESSI_PILOT_VERSION}" README.md
 
-    - name: verify if README.md is consistent with EESSI_CVMFS_REPO from init/eessi_defaults
-      run: |
-        source init/eessi_defaults
-        grep "${EESSI_CVMFS_REPO}" README.md
+        - name: verify if README.md is consistent with EESSI_CVMFS_REPO from init/eessi_defaults
+          run: |
+            source init/eessi_defaults
+            grep "${EESSI_CVMFS_REPO}" README.md

From 2e66057724c2a3b51b92e10c11edd910691d8075 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 15 Jan 2023 21:01:17 +0100
Subject: [PATCH 030/157] Fix hash for checkout action in README CI

---
 .github/workflows/tests_readme.yml | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/tests_readme.yml b/.github/workflows/tests_readme.yml
index 29dedcc3ff..5c6d0318d4 100644
--- a/.github/workflows/tests_readme.yml
+++ b/.github/workflows/tests_readme.yml
@@ -18,14 +18,15 @@ jobs:
   build:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v2
+        - name: Check out software-layer repository
+          uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
-    - name: verify if README.md is consistent with EESSI_PILOT_VERSION from init/eessi_defaults
-      run: |
-        source init/eessi_defaults
-        grep "${EESSI_PILOT_VERSION}" README.md
+        - name: verify if README.md is consistent with EESSI_PILOT_VERSION from init/eessi_defaults
+          run: |
+            source init/eessi_defaults
+            grep "${EESSI_PILOT_VERSION}" README.md
 
-    - name: verify if README.md is consistent with EESSI_CVMFS_REPO from init/eessi_defaults
-      run: |
-        source init/eessi_defaults
-        grep "${EESSI_CVMFS_REPO}" README.md
+        - name: verify if README.md is consistent with EESSI_CVMFS_REPO from init/eessi_defaults
+          run: |
+            source init/eessi_defaults
+            grep "${EESSI_CVMFS_REPO}" README.md

From f976b2233954eefcf0174b23fa4a823250f4538f Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 24 Jan 2023 14:01:08 +0100
Subject: [PATCH 031/157] restore original $PATH and $PYTHONPATH values after
 installing first EasyBuild module

---
 EESSI-pilot-install-software.sh | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
index 5646c09b03..e1b4089e2d 100755
--- a/EESSI-pilot-install-software.sh
+++ b/EESSI-pilot-install-software.sh
@@ -152,6 +152,10 @@ else
     pip_install_out=${TMPDIR}/pip_install.out
     pip3 install --prefix $EB_TMPDIR easybuild &> ${pip_install_out}
 
+    # keep track of original $PATH and $PYTHONPATH values, so we can restore them
+    ORIG_PATH=$PATH
+    ORIG_PYTHONPATH=$PYTHONPATH
+
     echo ">> Final installation in ${EASYBUILD_INSTALLPATH}..."
     export PATH=${EB_TMPDIR}/bin:$PATH
     export PYTHONPATH=$(ls -d ${EB_TMPDIR}/lib/python*/site-packages):$PYTHONPATH
@@ -161,6 +165,10 @@ else
     eb --install-latest-eb-release &> ${eb_install_out}
     check_exit_code $? "${ok_msg}" "${fail_msg}"
 
+    # restore origin $PATH and $PYTHONPATH values
+    export PATH=${ORIG_PATH}
+    export PYTHONPATH=${ORIG_PYTHONPATH}
+
     eb --search EasyBuild-${REQ_EB_VERSION}.eb | grep EasyBuild-${REQ_EB_VERSION}.eb > /dev/null
     if [[ $? -eq 0 ]]; then
         ok_msg="EasyBuild v${REQ_EB_VERSION} installed, alright!"

From 447bb2d0da7f6b41a149344eb055ed16ca188663 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Tue, 24 Jan 2023 21:59:41 +0100
Subject: [PATCH 032/157] added a function to obtain full path to a tool

---
 utils.sh | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/utils.sh b/utils.sh
index 5d8455bb68..cd7f758a0d 100644
--- a/utils.sh
+++ b/utils.sh
@@ -31,3 +31,35 @@ function check_exit_code {
         fatal_error "${fail_msg}"
     fi
 }
+
+function get_path_for_tool {
+    tool_name=$1
+    tool_envvar_name=$2
+
+    which_out=$(which ${tool_name} 2>&1)
+    exit_code=$?
+    if [[ ${exit_code} -eq 0 ]]; then
+        echo "INFO: found tool ${tool_name} in PATH (${which_out})" >&2
+        echo "${which_out}"
+        return 0
+    fi
+    if [[ -z "${tool_envvar_name}" ]]; then
+        msg="no env var holding the full path to tool '${tool_name}' provided"
+        echo "${msg}" >&2
+        return 1
+    else
+        tool_envvar_value=${!tool_envvar_name}
+        if [[ -x "${tool_envvar_value}" ]]; then
+            msg="INFO: found tool ${tool_envvar_value} via env var ${tool_envvar_name}"
+            echo "${msg}" >&2
+            echo "${tool_envvar_value}"
+            return 0
+        else
+            msg="ERROR: tool '${tool_name}' not in PATH\n"
+            msg+="ERROR: tool '${tool_envvar_value}' via '${tool_envvar_name}' not in PATH"
+            echo "${msg}" >&2
+            echo ""
+            return 2
+        fi
+    fi
+}

From e2c24cf757b912aef167364aa34d0abf723b7b03 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Tue, 24 Jan 2023 22:01:13 +0100
Subject: [PATCH 033/157] partial bot/build.sh for eessi_container.sh

---
 bot/build.sh | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 78 insertions(+)
 create mode 100755 bot/build.sh

diff --git a/bot/build.sh b/bot/build.sh
new file mode 100755
index 0000000000..90753db8a8
--- /dev/null
+++ b/bot/build.sh
@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+#
+# script to build the EESSI software layer. Intended use is that it is called
+# by a (batch) job running on a compute node.
+#
+# This script is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Thomas Roeblitz (@trz42)
+#
+# license: GPLv2
+#
+
+# ASSUMPTIONs:
+#  - working directory has been prepared by the bot with a checkout of a
+#    pull request (OR by some other means)
+#  - the working directory contains a directory 'cfg' where the main config
+#    file 'job.cfg' has been deposited
+#  - the directory may contain any additional files references in job.cfg
+#  - the tool 'yq' for working with json files is available via the PATH or
+#    the environment variable BOT_YQ (see https://github.com/mikefarah/yq)
+
+# defaults
+export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
+
+# source utils.sh
+source utils.sh
+
+# check setup / define key variables
+# get path for 'yq' (if not found, an empty string is returned)
+YQ=$(get_path_for_tool "yq" "BOT_YQ")
+exit_code=$?
+if [[ ${exit_code} -ne 0 ]]; then
+    fatal_error "could not find path to 'yq'; exiting"
+else
+    echo_green "found yq (${YQ})"
+fi
+
+# check if './cfg/job.cfg' exists
+if [[ ! -r "${JOB_CFG_FILE}" ]]; then
+    fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable"
+fi
+echo "obtaining configuration settings from '${JOB_CFG_FILE}'"
+
+LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' < ${JOB_CFG_FILE})
+echo "LOCAL_TMP='${LOCAL_TMP}'"
+echo -n "setting \$storage by replacing any var in '${LOCAL_TMP}' -> "
+# replace any env variable in ${LOCAL_TMP} with its
+#   current value (e.g., a value that is local to the job)
+storage=$(envsubst <<< ${LOCAL_TMP})
+echo "'${storage}'"
+
+
+# singularity/apptainer settings: load_modules, HOME, TMPDIR, BIND 
+LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' < ${JOB_CFG_FILE})
+echo "LOAD_MODULES='${LOAD_MODULES}'"
+
+export SINGULARITY_HOME="$(pwd):/eessi_bot_job"
+export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir"
+mkdir -p ${SINGULARITY_TMPDIR}
+
+if [[ ${storage} != /tmp* ]] ;
+then
+    export SINGULARITY_BIND="${storage}:/tmp"
+fi
+echo "SINGULARITY_BIND='${SINGULARITY_BIND}'"
+
+# load modules LOAD_MODULES is not empty
+if [[ ! -z ${LOAD_MODULES} ]]; then
+    for mod in $(echo ${LOAD_MODULES} | tr ',' '\n')
+    do
+        echo "bot/build.sh: loading module '${mod}'"
+        module load ${mod}
+    done
+else
+    echo "bot/build.sh: no modules to be loaded"
+fi
+

From 8cac3dfeb43a59f841f33baa62330bb38fe29f01 Mon Sep 17 00:00:00 2001
From: Michael Huebner <huebner@physik.uni-bonn.de>
Date: Wed, 25 Jan 2023 14:37:33 +0100
Subject: [PATCH 034/157] Update of where utils.sh is located when sourcing

---
 check_missing_installations.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/check_missing_installations.sh b/check_missing_installations.sh
index 30f9cc6ff7..e4c75aebd8 100755
--- a/check_missing_installations.sh
+++ b/check_missing_installations.sh
@@ -16,7 +16,7 @@ fi
 
 LOCAL_TMPDIR=$(mktemp -d)
 
-source $TOPDIR/utils.sh
+source $TOPDIR/scripts/utils.sh
 
 source $TOPDIR/configure_easybuild
 

From 3390a2d56ab5c28200ff487e124c74e87945679c Mon Sep 17 00:00:00 2001
From: Michael Huebner <huebner@physik.uni-bonn.de>
Date: Wed, 25 Jan 2023 14:40:06 +0100
Subject: [PATCH 035/157] Add create_directory_tarballs.sh to github CI

---
 .github/workflows/tests_scripts.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml
index dbf8933555..53f4bd13be 100644
--- a/.github/workflows/tests_scripts.yml
+++ b/.github/workflows/tests_scripts.yml
@@ -8,6 +8,7 @@ on:
       - run_in_compat_layer_env.sh
       - scripts/utils.sh
       - update_lmod_cache.sh
+      - create_directory_tarballs.sh
 
   pull_request:
     branches:
@@ -18,6 +19,7 @@ on:
       - run_in_compat_layer_env.sh
       - scripts/utils.sh
       - update_lmod_cache.sh
+      - create_directory_tarballs.sh
 permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:

From a71f8e4f79d53f5be848d5f7064ce3569c4e0477 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= <thomas.roblitz@uib.no>
Date: Wed, 25 Jan 2023 19:37:55 +0100
Subject: [PATCH 036/157] added function to determine IPv4 address

---
 utils.sh | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/utils.sh b/utils.sh
index cd7f758a0d..db533adc12 100644
--- a/utils.sh
+++ b/utils.sh
@@ -63,3 +63,12 @@ function get_path_for_tool {
         fi
     fi
 }
+
+function get_ipv4_address {
+    hname=$1
+    hipv4=$(grep ${hname} /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1)
+    # TODO try other methods if the one above does not work --> tool that verifies
+    #      what method can be used?
+    echo "${hipv4}"
+    return 0
+}

From b5bf008e4136b7877e06ca0602b6fd6aa49128c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= <thomas.roblitz@uib.no>
Date: Wed, 25 Jan 2023 19:39:53 +0100
Subject: [PATCH 037/157] various updates for bot/build.sh and
 eessi_container.sh

---
 bot/build.sh       |  69 ++++++++++++++++++++++----
 eessi_container.sh | 117 ++++++++++++++++++++++++---------------------
 2 files changed, 121 insertions(+), 65 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 90753db8a8..e7cc7983bb 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -42,30 +42,37 @@ if [[ ! -r "${JOB_CFG_FILE}" ]]; then
 fi
 echo "obtaining configuration settings from '${JOB_CFG_FILE}'"
 
-LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' < ${JOB_CFG_FILE})
+# if http_proxy is in cfg/job.cfg use it, if not use env var $http_proxy
+HTTP_PROXY=$(${YQ} '.site_config.http_proxy // "${http_proxy}"' ${JOB_CFG_FILE})
+echo "HTTP_PROXY='${HTTP_PROXY}'"
+
+LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE})
 echo "LOCAL_TMP='${LOCAL_TMP}'"
-echo -n "setting \$storage by replacing any var in '${LOCAL_TMP}' -> "
+# TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided
+
+echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> "
 # replace any env variable in ${LOCAL_TMP} with its
 #   current value (e.g., a value that is local to the job)
-storage=$(envsubst <<< ${LOCAL_TMP})
-echo "'${storage}'"
-
+STORAGE=$(envsubst <<< ${LOCAL_TMP})
+echo "'${STORAGE}'"
 
-# singularity/apptainer settings: load_modules, HOME, TMPDIR, BIND 
-LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' < ${JOB_CFG_FILE})
+# obtain list of modules to be loaded
+LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' ${JOB_CFG_FILE})
 echo "LOAD_MODULES='${LOAD_MODULES}'"
 
+# singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND
+CONTAINER=$(${YQ} '.site_config.container // ""' ${JOB_CFG_FILE})
 export SINGULARITY_HOME="$(pwd):/eessi_bot_job"
 export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir"
 mkdir -p ${SINGULARITY_TMPDIR}
 
-if [[ ${storage} != /tmp* ]] ;
+if [[ ${STORAGE} != /tmp* ]] ;
 then
-    export SINGULARITY_BIND="${storage}:/tmp"
+    export SINGULARITY_BIND="${STORAGE}:/tmp"
 fi
 echo "SINGULARITY_BIND='${SINGULARITY_BIND}'"
 
-# load modules LOAD_MODULES is not empty
+# load modules if LOAD_MODULES is not empty
 if [[ ! -z ${LOAD_MODULES} ]]; then
     for mod in $(echo ${LOAD_MODULES} | tr ',' '\n')
     do
@@ -76,3 +83,45 @@ else
     echo "bot/build.sh: no modules to be loaded"
 fi
 
+# determine repository to be used from entry .repository in cfg/job.cfg
+REPOSITORY=$(${YQ} '.repository.repo_id // ""' ${JOB_CFG_FILE})
+EESSI_REPOS_CFG_FILE_OVERRIDE=$(${YQ} '.repository.repos_cfg_file // "cfg/repos.cfg"' ${JOB_CFG_FILE})
+
+# determine architecture to be used from entry .architecture in cfg/job.cfg
+# default: leave empty to let downstream script(s) determine subdir to be used
+EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE})
+
+source init/minimal_eessi_env
+
+# TODO
+#   - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh
+#   - add handling of http(s)_proxy to eessi_container.sh, in there needs the
+#     CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust
+#     way to determine the IP address of a proxy)
+#   - bot needs to make repos.cfg and cfg_bundle available to job (likely, by copying
+#     files into './cfg/.' and defining '.repository.repos_cfg_file' in './cfg/job.cfg')
+
+# prepare options and directories for calling eessi_container.sh
+mkdir -p previous_tmp
+run_outerr=$(mktemp eessi_container.outerr.XXXXXXXXXX)
+CONTAINER_OPT=
+if [[ ! -z ${CONTAINER} ]]; then
+    CONTAINER_OPT="--container ${CONTAINER}"
+fi
+HTTP_PROXY_OPT=
+if [[ ! -z ${HTTP_PROXY} ]]; then
+    HTTP_PROXY_OPT="--container ${HTTP_PROXY}"
+fi
+REPOSITORY_OPT=
+if [[ ! -z ${REPOSITORY} ]]; then
+    REPOSITORY_OPT="--repository ${REPOSITORY}"
+fi
+./eessi_container.sh --access rw \
+                     ${CONTAINER_OPT} \
+                     ${HTTP_PROXY_OPT} \
+                     --info \
+                     --mode run \
+                     ${REPOSITORY_OPT} \
+                     --save $(pwd)/previous_tmp \
+                     --storage ${STORAGE} \
+                     ./install_software_layer.sh TODO installopts TODO commonopts TODO remainingopts "$@" 2>&1 | tee -a $(run_outerr)
diff --git a/eessi_container.sh b/eessi_container.sh
index 57b003368b..0016c8c9f0 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -38,8 +38,8 @@ MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5))
 REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6))
 RESUME_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7))
 SAVE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 8))
-#HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9))
-#HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10))
+HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9))
+HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10))
 RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11))
 
 # CernVM-FS settings
@@ -59,43 +59,43 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}"
 display_help() {
   echo "usage: $0 [OPTIONS] [SCRIPT]"
   echo " OPTIONS:"
-  echo "  -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]"
-  echo "  -c | --container IMG  - image file or URL defining the container to use"
-  echo "                          [default: docker://ghcr.io/eessi/build-node:debian10]"
-  echo "  -h | --help           - display this usage information [default: false]"
-  echo "  -g | --storage DIR    - directory space on host machine (used for"
-  echo "                          temporary data) [default: 1. TMPDIR, 2. /tmp]"
-  echo "  -m | --mode MODE      - with MODE==shell (launch interactive shell) or"
-  echo "                          MODE==run (run a script) [default: shell]"
-  echo "  -r | --repository CFG - configuration file or identifier defining the"
-  echo "                          repository to use [default: EESSI-pilot]"
-  echo "  -u | --resume DIR/TGZ - resume a previous run from a directory or tarball,"
-  echo "                          where DIR points to a previously used tmp directory"
-  echo "                          (check for output 'Using DIR as tmp ...' of a previous"
-  echo "                          run) and TGZ is the path to a tarball which is"
-  echo "                          unpacked the tmp dir stored on the local storage space"
-  echo "                          (see option --storage above) [default: not set]"
-  echo "  -s | --save DIR/TGZ   - save contents of tmp directory to a tarball in"
-  echo "                          directory DIR or provided with the fixed full path TGZ"
-  echo "                          when a directory is provided, the format of the"
-  echo "                          tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz"
-  echo "                          [default: not set]"
+  echo "  -a | --access {ro,rw}  - ro (read-only), rw (read & write) [default: ro]"
+  echo "  -c | --container IMG   - image file or URL defining the container to use"
+  echo "                           [default: docker://ghcr.io/eessi/build-node:debian11]"
+  echo "  -h | --help            - display this usage information [default: false]"
+  echo "  -i | --info            - display configuration information [default: false]"
+  echo "  -g | --storage DIR     - directory space on host machine (used for"
+  echo "                           temporary data) [default: 1. TMPDIR, 2. /tmp]"
+  echo "  -m | --mode MODE       - with MODE==shell (launch interactive shell) or"
+  echo "                           MODE==run (run a script) [default: shell]"
+  echo "  -r | --repository CFG  - configuration file or identifier defining the"
+  echo "                           repository to use [default: EESSI-pilot]"
+  echo "  -u | --resume DIR/TGZ  - resume a previous run from a directory or tarball,"
+  echo "                           where DIR points to a previously used tmp directory"
+  echo "                           (check for output 'Using DIR as tmp ...' of a previous"
+  echo "                           run) and TGZ is the path to a tarball which is"
+  echo "                           unpacked the tmp dir stored on the local storage space"
+  echo "                           (see option --storage above) [default: not set]"
+  echo "  -s | --save DIR/TGZ    - save contents of tmp directory to a tarball in"
+  echo "                           directory DIR or provided with the fixed full path TGZ"
+  echo "                           when a directory is provided, the format of the"
+  echo "                           tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz"
+  echo "                           [default: not set]"
+  echo "  -x | --http-proxy URL  - provides URL for the env variable http_proxy"
+  echo "                           [default: not set]; uses env var \$http_proxy if set"
+  echo "  -y | --https-proxy URL - provides URL for the env variable https_proxy"
+  echo "                           [default: not set]; uses env var \$https_proxy if set"
   echo
   echo " If value for --mode is 'run', the SCRIPT provided is executed."
   echo
   echo " FEATURES/OPTIONS to be implemented:"
-  echo "  -d | --dry-run           -  run script except for executing the container,"
-  echo "                              print information about setup [default: false]"
-  echo "  -i | --info              -  display configured repositories [default: false]"
-  #echo "  -x | --http-proxy URL    -  provides URL for the env variable http_proxy"
-  #echo "                              [default: not set]"
-  #echo "  -y | --https-proxy URL   -  provides URL for the env variable https_proxy"
-  #echo "                              [default: not set]"
+  echo "  -d | --dry-run         -  run script except for executing the container,"
+  echo "                            print information about setup [default: false]"
 }
 
 # set defaults for command line arguments
 ACCESS="ro"
-CONTAINER="docker://ghcr.io/eessi/build-node:debian10"
+CONTAINER="docker://ghcr.io/eessi/build-node:debian11"
 DRY_RUN=0
 INFO=0
 STORAGE=
@@ -103,8 +103,8 @@ MODE="shell"
 REPOSITORY="EESSI-pilot"
 RESUME=
 SAVE=
-#HTTP_PROXY=
-#HTTPS_PROXY=
+HTTP_PROXY=${http_proxy:-}
+HTTPS_PROXY=${https_proxy:-}
 RUN_SCRIPT_AND_ARGS=
 
 POSITIONAL_ARGS=()
@@ -151,16 +151,16 @@ while [[ $# -gt 0 ]]; do
       RESUME="$2"
       shift 2
       ;;
-#    -x|--http-proxy)
-#      HTTP_PROXY="$2"
-#      export http_proxy=${HTTP_PROXY}
-#      shift 2
-#      ;;
-#    -y|--https-proxy)
-#      HTTPS_PROXY="$2"
-#      export https_proxy=${HTTPS_PROXY}
-#      shift 2
-#      ;;
+    -x|--http-proxy)
+      HTTP_PROXY="$2"
+      export http_proxy=${HTTP_PROXY}
+      shift 2
+      ;;
+    -y|--https-proxy)
+      HTTPS_PROXY="$2"
+      export https_proxy=${HTTPS_PROXY}
+      shift 2
+      ;;
     -*|--*)
       fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}"
       ;;
@@ -281,26 +281,26 @@ fi
 # tmp dir for EESSI
 EESSI_TMPDIR=${EESSI_HOST_STORAGE}
 mkdir -p ${EESSI_TMPDIR}
-[[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}"
+[[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR='${EESSI_TMPDIR}'"
 
 # configure Singularity
 export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache
 mkdir -p ${SINGULARITY_CACHEDIR}
-[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}"
+[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'"
 
 # set env vars and create directories for CernVM-FS
 EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB}
 EESSI_CVMFS_VAR_RUN=${EESSI_TMPDIR}/${CVMFS_VAR_RUN}
 mkdir -p ${EESSI_CVMFS_VAR_LIB}
 mkdir -p ${EESSI_CVMFS_VAR_RUN}
-[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_LIB=${EESSI_CVMFS_VAR_LIB}"
-[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_RUN=${EESSI_CVMFS_VAR_RUN}"
+[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_LIB='${EESSI_CVMFS_VAR_LIB}'"
+[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_RUN='${EESSI_CVMFS_VAR_RUN}'"
 
 # allow that SINGULARITY_HOME is defined before script is run
 if [[ -z ${SINGULARITY_HOME} ]]; then
   export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}"
   mkdir -p ${EESSI_TMPDIR}/home
-  [[ ${INFO} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}"
+  [[ ${INFO} -eq 1 ]] && echo "SINGULARITY_HOME='${SINGULARITY_HOME}'"
 fi
 
 # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined)
@@ -308,9 +308,9 @@ BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/ru
 BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp"
 [[ ${INFO} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}"
 
-# set up repository config (always create cfg dir and populate it with info when
+# set up repository config (always create directory repos_cfg and populate it with info when
 # arg -r|--repository is used)
-mkdir -p ${EESSI_TMPDIR}/cfg
+mkdir -p ${EESSI_TMPDIR}/repos_cfg
 if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then
   # need to source defaults as late as possible (see other sourcing below)
   source ${base_dir}/init/eessi_defaults
@@ -367,13 +367,13 @@ else
 
   # only unpack config_bundle if we're not resuming from a previous run
   if [[ -z ${RESUME} ]]; then
-    tar xf ${config_bundle} -C ${EESSI_TMPDIR}/cfg
+    tar xf ${config_bundle} -C ${EESSI_TMPDIR}/repos_cfg
   fi
 
   for src in "${!cfg_file_map[@]}"
   do
     target=${cfg_file_map[${src}]}
-    BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/cfg/${src}:${target}"
+    BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}"
   done
   export EESSI_PILOT_VERSION_OVERRIDE=${repo_version}
   export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}"
@@ -397,8 +397,8 @@ if [[ "${ACCESS}" == "rw" ]]; then
   EESSI_CVMFS_OVERLAY_WORK=/tmp/overlay-work
   mkdir -p ${EESSI_TMPDIR}/overlay-upper
   mkdir -p ${EESSI_TMPDIR}/overlay-work
-  [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}"
-  [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}"
+  [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER='${EESSI_CVMFS_OVERLAY_UPPER}'"
+  [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK='${EESSI_CVMFS_OVERLAY_WORK}'"
 
   # set environment variables for fuse mounts in Singularity container
   export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}"
@@ -427,7 +427,14 @@ if [[ -z ${SINGULARITY_BIND} ]]; then
 else
     export SINGULARITY_BIND="${SINGULARITY_BIND},${BIND_PATHS}"
 fi
-[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}"
+[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_BIND='${SINGULARITY_BIND}'"
+
+# pass $EESSI_SOFTWARE_SUBDIR_OVERRIDE into build container (if set)
+if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
+    export SINGULARITYENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
+    # also specify via $APPTAINERENV_* (future proof, cfr. https://apptainer.org/docs/user/latest/singularity_compatibility.html#singularity-environment-variable-compatibility)
+    export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
+fi
 
 echo "Launching container with command (next line):"
 echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}"

From b5c07eeb99e77191d818e64788b0d419031cf526 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 27 Jan 2023 20:30:58 +0100
Subject: [PATCH 038/157] provide CVMFS_HTTP_PROXY if necessary

---
 bot/build.sh       |  3 ++-
 eessi_container.sh | 39 ++++++++++++++++++++++++++++-----------
 2 files changed, 30 insertions(+), 12 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index e7cc7983bb..a4f54cba3c 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -95,7 +95,8 @@ source init/minimal_eessi_env
 
 # TODO
 #   - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh
-#   - add handling of http(s)_proxy to eessi_container.sh, in there needs the
+#     TODO ensure that the bot makes use of that.
+#   - CODED add handling of http(s)_proxy to eessi_container.sh, in there needs the
 #     CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust
 #     way to determine the IP address of a proxy)
 #   - bot needs to make repos.cfg and cfg_bundle available to job (likely, by copying
diff --git a/eessi_container.sh b/eessi_container.sh
index 0016c8c9f0..c2b4988162 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -19,8 +19,8 @@
 # 2. set up host storage/tmp
 # 3. set up common vars and directories
 # 4. set up vars specific to a scenario
-# 5. initialize host storage/tmp from previous run if provided
-# 6. run container
+# 5. run container
+# 6. save tmp (if requested)
 
 # -. initial settings & exit codes
 base_dir=$(dirname $(realpath $0))
@@ -276,7 +276,7 @@ fi
 #      |-overlay-upper
 #      |-overlay-work
 #      |-home
-#      |-cfg
+#      |-repos_cfg
 
 # tmp dir for EESSI
 EESSI_TMPDIR=${EESSI_HOST_STORAGE}
@@ -358,9 +358,9 @@ else
   cfg_init_file_map "${config_map}"
   [[ ${INFO} -eq 1 ]] && cfg_print_map
 
-  # TODO use information to set up dir ${EESSI_TMPDIR}/cfg,
-  #      define BIND mounts and override repo name and version
-  # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/cfg
+  # use information to set up dir ${EESSI_TMPDIR}/repos_cfg,
+  #     define BIND mounts and override repo name and version
+  # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/repos_cfg
   if [[ ! -r ${config_bundle} ]]; then
     fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE}
   fi
@@ -381,6 +381,26 @@ else
   source ${base_dir}/init/eessi_defaults
 fi
 
+# if http_proxy is not empty, we assume that the machine accesses internet
+# via a proxy. then we need to add CVMFS_HTTP_PROXY to
+# ${EESSI_TMPDIR}/repos_cfg/default.local on host (and possibly add a BIND
+# MOUNT if it was not yet in BIND_PATHS)
+if [[ ! -z ${http_proxy} ]]; then
+    # TODO tolerate other formats for proxy URLs, for now assume format is
+    # http://SOME_HOSTNAME:SOME_PORT
+    PROXY_HOST_AND_PORT=${http_proxy#http:\/\//} # strip http://
+    PROXY_PORT=${PROXY_HOST_AND_PORT#.*:/} # remove hostname: to get port
+    HTTP_PROXY_HOSTNAME=${PROXY_HOST_AND_PORT%:${PROX_PORT}/}
+    HTTP_PROXY_IPV4=$(get_ipv4_address ${HTTP_PROXY_HOSTNAME}) 
+    echo "CVMFS_HTTP_PROXY=\"${http_proxy}|" \
+         "http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \
+       >> ${EESSI_TMPDIR}/repos_cfg/default.local
+    cat ${EESSI_TMPDIR}/repos_cfg/default.local
+    # if default.local is not BIND mounted into container, add it to BIND_PATHS
+    if [[ ${BIND_PATHS} !~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then
+        export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local"
+    fi
+fi
 
 # 4. set up vars and dirs specific to a scenario
 
@@ -417,10 +437,7 @@ if [[ "${ACCESS}" == "rw" ]]; then
 fi
 
 
-# 5. initialize host storage/tmp from previous run if provided
-
-
-# 6. run container
+# 5. run container
 # final settings
 if [[ -z ${SINGULARITY_BIND} ]]; then
     export SINGULARITY_BIND="${BIND_PATHS}"
@@ -442,7 +459,7 @@ echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_
 #      provided to the script
 singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}
 
-# 7. save tmp if requested (arg -s|--save)
+# 6. save tmp if requested (arg -s|--save)
 if [[ ! -z ${SAVE} ]]; then
   # Note, for now we don't try to be smart and record in any way the OS and
   #   ARCH which might have been used internally, eg, when software packages

From 4ff37218793559a6e3a8ea9cef58198e33b7dcdf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= <thomas.roblitz@uib.no>
Date: Fri, 27 Jan 2023 22:41:36 +0100
Subject: [PATCH 039/157] add functions to derive hostname and port number from
 proxy URL

---
 utils.sh | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/utils.sh b/utils.sh
index db533adc12..fb4eff1e3f 100644
--- a/utils.sh
+++ b/utils.sh
@@ -64,6 +64,30 @@ function get_path_for_tool {
     fi
 }
 
+function get_host_from_url {
+    url=$1
+    re="(http|https)://([^/:]+)"
+    if [[ $url =~ $re ]]; then
+        echo ${BASH_REMATCH[2]}
+        return 0
+    else
+        echo ""
+        return 1
+    fi 
+}
+
+function get_port_from_url {
+    url=$1
+    re="(http|https)://[^:]+:([0-9]+)"
+    if [[ $url =~ $re ]]; then
+        echo ${BASH_REMATCH[2]}
+        return 0
+    else
+        echo ""
+        return 1
+    fi 
+}
+
 function get_ipv4_address {
     hname=$1
     hipv4=$(grep ${hname} /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1)

From 7a58fbff63712d81dc7dd3206ae2d35535c58b24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= <thomas.roblitz@uib.no>
Date: Fri, 27 Jan 2023 22:43:27 +0100
Subject: [PATCH 040/157] add function to determine config sections

---
 cfg_files.sh | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/cfg_files.sh b/cfg_files.sh
index 885ebd0877..ffd29db50a 100644
--- a/cfg_files.sh
+++ b/cfg_files.sh
@@ -47,9 +47,7 @@ function cfg_load {
     else
       val=$(cfg_get_key_value $line)
       # trim leading and trailing spaces as well
-      #cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//')
       cur_key=$(echo $val | cut -f1 -d'=' | cfg_trim_spaces)
-      #cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//')
       cur_val=$(echo $val | cut -f2 -d'=' | cfg_trim_spaces)
       if [[ -n "$cur_key" ]]; then
         # section + key is the associative in bash array, the field separator is space
@@ -69,6 +67,20 @@ function cfg_print {
   done
 }
 
+function cfg_sections {
+  declare -A sections
+  for key in "${!cfg_repos[@]}"
+  do
+    # extract section from the associative key
+    section=$(echo $key | cut -f1 -d ' ')
+    sections[${section}]=1
+  done
+  for repo in "${!sections[@]}"
+  do
+      echo "${repo}"
+  done
+}
+
 function cfg_get_value {
   section=$1
   key=$2

From a5ea7d8aea2c92380839ecbf290c28d278c43835 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= <thomas.roblitz@uib.no>
Date: Fri, 27 Jan 2023 22:44:06 +0100
Subject: [PATCH 041/157] add parameter to list repositories, fix bugs

fixed bugs

- determining IP address for proxy host was wrong
- handling of positional parameters for executing commands with arguments was wrong
---
 eessi_container.sh | 43 ++++++++++++++++++++++++++++++-------------
 1 file changed, 30 insertions(+), 13 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index c2b4988162..d8d7cb2bdb 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -66,6 +66,7 @@ display_help() {
   echo "  -i | --info            - display configuration information [default: false]"
   echo "  -g | --storage DIR     - directory space on host machine (used for"
   echo "                           temporary data) [default: 1. TMPDIR, 2. /tmp]"
+  echo "  -l | --list-repos      - list available repository identifiers [default: false]"
   echo "  -m | --mode MODE       - with MODE==shell (launch interactive shell) or"
   echo "                           MODE==run (run a script) [default: shell]"
   echo "  -r | --repository CFG  - configuration file or identifier defining the"
@@ -99,13 +100,13 @@ CONTAINER="docker://ghcr.io/eessi/build-node:debian11"
 DRY_RUN=0
 INFO=0
 STORAGE=
+LIST_REPOS=0
 MODE="shell"
 REPOSITORY="EESSI-pilot"
 RESUME=
 SAVE=
 HTTP_PROXY=${http_proxy:-}
 HTTPS_PROXY=${https_proxy:-}
-RUN_SCRIPT_AND_ARGS=
 
 POSITIONAL_ARGS=()
 
@@ -135,6 +136,10 @@ while [[ $# -gt 0 ]]; do
       INFO=1
       shift 1
       ;;
+    -l|--list-repos)
+      LIST_REPOS=1
+      shift 1
+      ;;
     -m|--mode)
       MODE="$2"
       shift 2
@@ -173,6 +178,17 @@ done
 
 set -- "${POSITIONAL_ARGS[@]}"
 
+if [[ ${LIST_REPOS} -eq 1 ]]; then
+    echo "Repositories defined in the config file '${EESSI_REPOS_CFG_FILE}':"
+    echo "    EESSI-pilot [default]"
+    cfg_load ${EESSI_REPOS_CFG_FILE}
+    sections=$(cfg_sections)
+    while IFS= read -r repo_id
+    do
+        echo "    ${repo_id}"
+    done <<< "${sections}"
+    exit 0
+fi
 
 # 1. check if argument values are valid
 # (arg -a|--access) check if ACCESS is supported
@@ -214,8 +230,6 @@ fi
 if [[ "${MODE}" == "run" ]]; then
   if [[ $# -eq 0 ]]; then
     fatal_error "no command specified to run?!" "${RUN_SCRIPT_MISSING_EXITCODE}"
-  else
-    RUN_SCRIPT_AND_ARGS=$@
   fi
 fi
 
@@ -387,17 +401,20 @@ fi
 # MOUNT if it was not yet in BIND_PATHS)
 if [[ ! -z ${http_proxy} ]]; then
     # TODO tolerate other formats for proxy URLs, for now assume format is
-    # http://SOME_HOSTNAME:SOME_PORT
-    PROXY_HOST_AND_PORT=${http_proxy#http:\/\//} # strip http://
-    PROXY_PORT=${PROXY_HOST_AND_PORT#.*:/} # remove hostname: to get port
-    HTTP_PROXY_HOSTNAME=${PROXY_HOST_AND_PORT%:${PROX_PORT}/}
-    HTTP_PROXY_IPV4=$(get_ipv4_address ${HTTP_PROXY_HOSTNAME}) 
-    echo "CVMFS_HTTP_PROXY=\"${http_proxy}|" \
-         "http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \
+    # http://SOME_HOSTNAME:SOME_PORT/
+    [[ ${INFO} -eq 1 ]] && echo "http_proxy='${http_proxy}'"
+    PROXY_HOST=$(get_host_from_url ${http_proxy})
+    [[ ${INFO} -eq 1 ]] && echo "PROXY_HOST='${PROXY_HOST}'"
+    PROXY_PORT=$(get_port_from_url ${http_proxy})
+    [[ ${INFO} -eq 1 ]] && echo "PROXY_PORT='${PROXY_PORT}'"
+    HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST}) 
+    [[ ${INFO} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'"
+    echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \
        >> ${EESSI_TMPDIR}/repos_cfg/default.local
     cat ${EESSI_TMPDIR}/repos_cfg/default.local
+
     # if default.local is not BIND mounted into container, add it to BIND_PATHS
-    if [[ ${BIND_PATHS} !~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then
+    if [[ ! ${BIND_PATHS} =~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then
         export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local"
     fi
 fi
@@ -454,10 +471,10 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
 fi
 
 echo "Launching container with command (next line):"
-echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}"
+echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
 # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level
 #      provided to the script
-singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}
+singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"
 
 # 6. save tmp if requested (arg -s|--save)
 if [[ ! -z ${SAVE} ]]; then

From 6ed3d85ff5f11bca37f838c533320733c3ddaf08 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 29 Jan 2023 14:27:26 +0100
Subject: [PATCH 042/157] fix bash pattern matching operator

---
 eessi_container.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index c2b4988162..0cab47e8fb 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -397,7 +397,7 @@ if [[ ! -z ${http_proxy} ]]; then
        >> ${EESSI_TMPDIR}/repos_cfg/default.local
     cat ${EESSI_TMPDIR}/repos_cfg/default.local
     # if default.local is not BIND mounted into container, add it to BIND_PATHS
-    if [[ ${BIND_PATHS} !~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then
+    if [[ ! ${BIND_PATHS} =~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then
         export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local"
     fi
 fi

From fd5cc3d339bffd7db248c43be909d195db8a64bd Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 29 Jan 2023 14:29:25 +0100
Subject: [PATCH 043/157] various updates to bot/build.sh

- added handling of HTTPS_PROXY
- moved CONTAINER setting from site_config to repository section (different
  repositories could require different containers)
- added handling of CPU_TARGET and EESSI_SOFTWARE_SUBDIR_OVERRIDE
- updated TODOs
- added setting of EESSI_REPOS_CFG_FILE_OVERRIDE
- bug fixes
---
 bot/build.sh | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index a4f54cba3c..3d06477a15 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -46,6 +46,10 @@ echo "obtaining configuration settings from '${JOB_CFG_FILE}'"
 HTTP_PROXY=$(${YQ} '.site_config.http_proxy // "${http_proxy}"' ${JOB_CFG_FILE})
 echo "HTTP_PROXY='${HTTP_PROXY}'"
 
+# if https_proxy is in cfg/job.cfg use it, if not use env var $https_proxy
+HTTPS_PROXY=$(${YQ} '.site_config.https_proxy // "${https_proxy}"' ${JOB_CFG_FILE})
+echo "HTTPS_PROXY='${HTTPS_PROXY}'"
+
 LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE})
 echo "LOCAL_TMP='${LOCAL_TMP}'"
 # TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided
@@ -61,7 +65,7 @@ LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' ${JOB_CFG_FILE})
 echo "LOAD_MODULES='${LOAD_MODULES}'"
 
 # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND
-CONTAINER=$(${YQ} '.site_config.container // ""' ${JOB_CFG_FILE})
+CONTAINER=$(${YQ} '.repository.container // ""' ${JOB_CFG_FILE})
 export SINGULARITY_HOME="$(pwd):/eessi_bot_job"
 export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir"
 mkdir -p ${SINGULARITY_TMPDIR}
@@ -89,13 +93,19 @@ EESSI_REPOS_CFG_FILE_OVERRIDE=$(${YQ} '.repository.repos_cfg_file // "cfg/repos.
 
 # determine architecture to be used from entry .architecture in cfg/job.cfg
 # default: leave empty to let downstream script(s) determine subdir to be used
-EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE})
+if [[ ! -z "${CPU_TARGET}" ]]; then
+    EESSI_SOFTWARE_SUBDIR_OVERRIDE=${CPU_TARGET}
+else
+    EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE})
+fi
 
 source init/minimal_eessi_env
 
 # TODO
 #   - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh
-#     TODO ensure that the bot makes use of that.
+#     TODO ensure that the bot makes use of that. (currently sets env var
+#     CPU_TARGET & adds --export=ALL,CPU_TARGET=val to sbatch command ... also
+#     add it to cfg/job.cfg - .architecture.software_subdir)
 #   - CODED add handling of http(s)_proxy to eessi_container.sh, in there needs the
 #     CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust
 #     way to determine the IP address of a proxy)
@@ -111,18 +121,25 @@ if [[ ! -z ${CONTAINER} ]]; then
 fi
 HTTP_PROXY_OPT=
 if [[ ! -z ${HTTP_PROXY} ]]; then
-    HTTP_PROXY_OPT="--container ${HTTP_PROXY}"
+    HTTP_PROXY_OPT="--http-proxy ${HTTP_PROXY}"
+fi
+HTTPS_PROXY_OPT=
+if [[ ! -z ${HTTPS_PROXY} ]]; then
+    HTTPS_PROXY_OPT="--https-proxy ${HTTPS_PROXY}"
 fi
 REPOSITORY_OPT=
 if [[ ! -z ${REPOSITORY} ]]; then
     REPOSITORY_OPT="--repository ${REPOSITORY}"
 fi
+# set EESSI_REPOS_CFG_FILE_OVERRIDE to ./cfg/repos.cfg
+export EESSI_REPOS_CFG_FILE_OVERRIDE=$(pwd)/cfg/repos.cfg
 ./eessi_container.sh --access rw \
                      ${CONTAINER_OPT} \
                      ${HTTP_PROXY_OPT} \
+                     ${HTTPS_PROXY_OPT} \
                      --info \
                      --mode run \
                      ${REPOSITORY_OPT} \
                      --save $(pwd)/previous_tmp \
                      --storage ${STORAGE} \
-                     ./install_software_layer.sh TODO installopts TODO commonopts TODO remainingopts "$@" 2>&1 | tee -a $(run_outerr)
+                     ./install_software_layer.sh TODO installopts TODO commonopts TODO remainingopts "$@" 2>&1 | tee -a ${run_outerr}

From b2eab080cba313c72e2c72f91429242b1bf04990 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 29 Jan 2023 19:10:40 +0100
Subject: [PATCH 044/157] replace FILE with DIR in
 EESSI_REPOS_CFG_FILE_OVERRIDE

---
 bot/build.sh       | 8 ++++----
 eessi_container.sh | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 3d06477a15..7de3594d41 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -89,7 +89,7 @@ fi
 
 # determine repository to be used from entry .repository in cfg/job.cfg
 REPOSITORY=$(${YQ} '.repository.repo_id // ""' ${JOB_CFG_FILE})
-EESSI_REPOS_CFG_FILE_OVERRIDE=$(${YQ} '.repository.repos_cfg_file // "cfg/repos.cfg"' ${JOB_CFG_FILE})
+EESSI_REPOS_CFG_DIR_OVERRIDE=$(${YQ} '.repository.repos_cfg_dir // "cfg"' ${JOB_CFG_FILE})
 
 # determine architecture to be used from entry .architecture in cfg/job.cfg
 # default: leave empty to let downstream script(s) determine subdir to be used
@@ -131,8 +131,8 @@ REPOSITORY_OPT=
 if [[ ! -z ${REPOSITORY} ]]; then
     REPOSITORY_OPT="--repository ${REPOSITORY}"
 fi
-# set EESSI_REPOS_CFG_FILE_OVERRIDE to ./cfg/repos.cfg
-export EESSI_REPOS_CFG_FILE_OVERRIDE=$(pwd)/cfg/repos.cfg
+# set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
+export EESSI_REPOS_CFG_DIR_OVERRIDE=$(pwd)/cfg
 ./eessi_container.sh --access rw \
                      ${CONTAINER_OPT} \
                      ${HTTP_PROXY_OPT} \
@@ -142,4 +142,4 @@ export EESSI_REPOS_CFG_FILE_OVERRIDE=$(pwd)/cfg/repos.cfg
                      ${REPOSITORY_OPT} \
                      --save $(pwd)/previous_tmp \
                      --storage ${STORAGE} \
-                     ./install_software_layer.sh TODO installopts TODO commonopts TODO remainingopts "$@" 2>&1 | tee -a ${run_outerr}
+                     ./install_software_layer.sh "$@" 2>&1 | tee -a ${run_outerr}
diff --git a/eessi_container.sh b/eessi_container.sh
index d8d7cb2bdb..4ca494e136 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -47,8 +47,8 @@ CVMFS_VAR_LIB="var-lib-cvmfs"
 CVMFS_VAR_RUN="var-run-cvmfs"
 
 # repository cfg file, default name (default location: $PWD)
-#   can be overwritten by setting env var EESSI_REPOS_CFG_FILE_OVERRIDE
-export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}"
+#   can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE
+export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=.}/repos.cfg"
 
 
 # 0. parse args

From 7e200a185782727ad61f7c1e78f4b52228300a4d Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 29 Jan 2023 19:56:41 +0100
Subject: [PATCH 045/157] fix whitespace issues

---
 cfg_files.sh        | 1 -
 eessi_container.sh  | 2 +-
 init/eessi_defaults | 1 -
 utils.sh            | 4 ++--
 4 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/cfg_files.sh b/cfg_files.sh
index ffd29db50a..57ea2f7c03 100644
--- a/cfg_files.sh
+++ b/cfg_files.sh
@@ -165,4 +165,3 @@ function cfg_print_map {
     echo "${index} --> ${cfg_file_map[${index}]}"
   done
 }
-
diff --git a/eessi_container.sh b/eessi_container.sh
index 4ca494e136..3a9fac2955 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -407,7 +407,7 @@ if [[ ! -z ${http_proxy} ]]; then
     [[ ${INFO} -eq 1 ]] && echo "PROXY_HOST='${PROXY_HOST}'"
     PROXY_PORT=$(get_port_from_url ${http_proxy})
     [[ ${INFO} -eq 1 ]] && echo "PROXY_PORT='${PROXY_PORT}'"
-    HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST}) 
+    HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST})
     [[ ${INFO} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'"
     echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \
        >> ${EESSI_TMPDIR}/repos_cfg/default.local
diff --git a/init/eessi_defaults b/init/eessi_defaults
index 1b5ce07fb1..f482cbc269 100644
--- a/init/eessi_defaults
+++ b/init/eessi_defaults
@@ -10,4 +10,3 @@
 
 export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/pilot.eessi-hpc.org}"
 export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2021.12}"
-
diff --git a/utils.sh b/utils.sh
index fb4eff1e3f..d0da95e87f 100644
--- a/utils.sh
+++ b/utils.sh
@@ -73,7 +73,7 @@ function get_host_from_url {
     else
         echo ""
         return 1
-    fi 
+    fi
 }
 
 function get_port_from_url {
@@ -85,7 +85,7 @@ function get_port_from_url {
     else
         echo ""
         return 1
-    fi 
+    fi
 }
 
 function get_ipv4_address {

From 57b6f27191ffce26c05065b1ce2a7e0841309e4d Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 30 Jan 2023 17:48:52 +0100
Subject: [PATCH 046/157] fix handling of HTTP(S)_PROXY settings

---
 bot/build.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 7de3594d41..d82048e86f 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -43,11 +43,13 @@ fi
 echo "obtaining configuration settings from '${JOB_CFG_FILE}'"
 
 # if http_proxy is in cfg/job.cfg use it, if not use env var $http_proxy
-HTTP_PROXY=$(${YQ} '.site_config.http_proxy // "${http_proxy}"' ${JOB_CFG_FILE})
+HTTP_PROXY=$(${YQ} '.site_config.http_proxy // ""' ${JOB_CFG_FILE})
+HTTP_PROXY=${HTTP_PROXY:-${http_proxy}}
 echo "HTTP_PROXY='${HTTP_PROXY}'"
 
 # if https_proxy is in cfg/job.cfg use it, if not use env var $https_proxy
-HTTPS_PROXY=$(${YQ} '.site_config.https_proxy // "${https_proxy}"' ${JOB_CFG_FILE})
+HTTPS_PROXY=$(${YQ} '.site_config.https_proxy // ""' ${JOB_CFG_FILE})
+HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}}
 echo "HTTPS_PROXY='${HTTPS_PROXY}'"
 
 LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE})

From 4d5cbce88d81d612508f6ec43d35b2f69fca71ac Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 1 Feb 2023 14:00:44 +0100
Subject: [PATCH 047/157] remove incorrect BUILD_CONTAINER line in
 run_in_compat_layer_env.sh

---
 run_in_compat_layer_env.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh
index e4850d677d..be01bcd201 100755
--- a/run_in_compat_layer_env.sh
+++ b/run_in_compat_layer_env.sh
@@ -3,7 +3,6 @@
 base_dir=$(dirname $(realpath $0))
 source ${base_dir}/init/eessi_defaults
 
-BUILD_CONTAINER="docker://ghcr.io/eessi/build-node:debian10"
 if [ -z $EESSI_PILOT_VERSION ]; then
     echo "ERROR: \$EESSI_PILOT_VERSION must be set!" >&2
     exit 1

From f513b120b7264c0a4af1479feac533d86b95b24f Mon Sep 17 00:00:00 2001
From: Michael Huebner <huebner@physik.uni-bonn.de>
Date: Thu, 2 Feb 2023 15:35:42 +0100
Subject: [PATCH 048/157] Add CI test for the create_directory_tarballs.sh
 script

---
 .github/workflows/tests_scripts.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml
index 53f4bd13be..a78c71dc9a 100644
--- a/.github/workflows/tests_scripts.yml
+++ b/.github/workflows/tests_scripts.yml
@@ -51,3 +51,13 @@ jobs:
           # force using x86_64/generic, to avoid triggering an installation from scratch
           sed -i "s@./EESSI-pilot-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-pilot-install-software.sh\"@g" install_software_layer.sh
           ./build_container.sh run /tmp/$USER/EESSI /tmp/install_software_layer.sh
+
+    - name: test create_directory_tarballs.sh script
+      run: |
+          # scripts need to be copied to /tmp,
+          # since create_directory_tarballs.sh must be accessible from within build container
+          cp -a * /tmp/
+          cd /tmp
+          ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh
+          # check if tarballs have been produced
+          ls -l *.tar.gz

From e4702ee06c71fba563e9d4f362fdf3aac41d394f Mon Sep 17 00:00:00 2001
From: Michael Huebner <huebner@physik.uni-bonn.de>
Date: Thu, 2 Feb 2023 17:47:24 +0100
Subject: [PATCH 049/157] Need to specify the EESSI version to test
 create_directory_tarballs

---
 .github/workflows/tests_scripts.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml
index a78c71dc9a..23bb36a7e8 100644
--- a/.github/workflows/tests_scripts.yml
+++ b/.github/workflows/tests_scripts.yml
@@ -58,6 +58,6 @@ jobs:
           # since create_directory_tarballs.sh must be accessible from within build container
           cp -a * /tmp/
           cd /tmp
-          ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh
+          ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh 2021.12
           # check if tarballs have been produced
           ls -l *.tar.gz

From c4b9cf5e2a4dcd75fca54e9fc9da2c6b69011d9e Mon Sep 17 00:00:00 2001
From: Michael Huebner <huebner@physik.uni-bonn.de>
Date: Thu, 2 Feb 2023 18:15:34 +0100
Subject: [PATCH 050/157] Fix wildcard usage when building tarball of the
 scripts dir

---
 create_directory_tarballs.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/create_directory_tarballs.sh b/create_directory_tarballs.sh
index c254c2c229..be282463eb 100755
--- a/create_directory_tarballs.sh
+++ b/create_directory_tarballs.sh
@@ -42,7 +42,7 @@ echo_green "Done! Created tarball ${tarname}."
 tartmp=$(mktemp -t -d scripts.XXXXX)
 mkdir "${tartmp}/${version}"
 tarname="eessi-${version}-scripts-$(date +%s).tar.gz"
-curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --strip-components=1 --wildcards */scripts/
+curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --strip-components=1 --wildcards '*/scripts/'
 tar czf "${tarname}" -C "${tartmp}" "${version}"
 rm -rf "${tartmp}"
 

From fce504fb466be005f1d417e96e47f442c8f29c52 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 3 Feb 2023 11:54:47 +0100
Subject: [PATCH 051/157] include changes made to test PR for building with the
 bot

---
 bot/build.sh               | 87 ++++++++++++++++++++++++++++++++------
 eessi_container.sh         | 50 +++++++++++++++-------
 run_in_compat_layer_env.sh | 15 ++++++-
 3 files changed, 121 insertions(+), 31 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index d82048e86f..8565511bfa 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -72,12 +72,6 @@ export SINGULARITY_HOME="$(pwd):/eessi_bot_job"
 export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir"
 mkdir -p ${SINGULARITY_TMPDIR}
 
-if [[ ${STORAGE} != /tmp* ]] ;
-then
-    export SINGULARITY_BIND="${STORAGE}:/tmp"
-fi
-echo "SINGULARITY_BIND='${SINGULARITY_BIND}'"
-
 # load modules if LOAD_MODULES is not empty
 if [[ ! -z ${LOAD_MODULES} ]]; then
     for mod in $(echo ${LOAD_MODULES} | tr ',' '\n')
@@ -91,7 +85,20 @@ fi
 
 # determine repository to be used from entry .repository in cfg/job.cfg
 REPOSITORY=$(${YQ} '.repository.repo_id // ""' ${JOB_CFG_FILE})
-EESSI_REPOS_CFG_DIR_OVERRIDE=$(${YQ} '.repository.repos_cfg_dir // "cfg"' ${JOB_CFG_FILE})
+EESSI_REPOS_CFG_DIR_OVERRIDE=$(${YQ} '.repository.repos_cfg_dir // ""' ${JOB_CFG_FILE})
+export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg}
+
+# determine pilot version to be used from .repository.repo_version in cfg/job.cfg
+# here, just set & export EESSI_PILOT_VERSION_OVERRIDE
+# next script (eessi_container.sh) makes use of it via sourcing init scripts
+# (e.g., init/eessi_defaults or init/minimal_eessi_env)
+export EESSI_PILOT_VERSION_OVERRIDE=$(${YQ} '.repository.repo_version // ""' ${JOB_CFG_FILE})
+
+# determine CVMFS repo to be used from .repository.repo_name in cfg/job.cfg
+# here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down
+# "source init/eessi_defaults" via sourcing init/minimal_eessi_env
+export EESSI_CVMFS_REPO_OVERRIDE=$(${YQ} '.repository.repo_name // ""' ${JOB_CFG_FILE})
+
 
 # determine architecture to be used from entry .architecture in cfg/job.cfg
 # default: leave empty to let downstream script(s) determine subdir to be used
@@ -100,8 +107,11 @@ if [[ ! -z "${CPU_TARGET}" ]]; then
 else
     EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE})
 fi
+export EESSI_SOFTWARE_SUBDIR_OVERRIDE
 
-source init/minimal_eessi_env
+# get EESSI_OS_TYPE from .architecture.os_type in cfg/job.cfg (default: linux)
+EESSI_OS_TYPE=$(${YQ} '.architecture.os_type // ""' ${JOB_CFG_FILE})
+export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux}
 
 # TODO
 #   - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh
@@ -112,11 +122,9 @@ source init/minimal_eessi_env
 #     CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust
 #     way to determine the IP address of a proxy)
 #   - bot needs to make repos.cfg and cfg_bundle available to job (likely, by copying
-#     files into './cfg/.' and defining '.repository.repos_cfg_file' in './cfg/job.cfg')
+#     files into './cfg/.' and defining '.repository.repos_cfg_dir' in './cfg/job.cfg')
 
 # prepare options and directories for calling eessi_container.sh
-mkdir -p previous_tmp
-run_outerr=$(mktemp eessi_container.outerr.XXXXXXXXXX)
 CONTAINER_OPT=
 if [[ ! -z ${CONTAINER} ]]; then
     CONTAINER_OPT="--container ${CONTAINER}"
@@ -133,8 +141,21 @@ REPOSITORY_OPT=
 if [[ ! -z ${REPOSITORY} ]]; then
     REPOSITORY_OPT="--repository ${REPOSITORY}"
 fi
+mkdir -p previous_tmp
+build_outerr=$(mktemp build.outerr.XXXX)
+echo "Executing command to build software:"
+echo "./eessi_container.sh --access rw"
+echo "                     ${CONTAINER_OPT}"
+echo "                     ${HTTP_PROXY_OPT}"
+echo "                     ${HTTPS_PROXY_OPT}"
+echo "                     --info"
+echo "                     --mode run"
+echo "                     ${REPOSITORY_OPT}"
+echo "                     --save ${PWD}/previous_tmp"
+echo "                     --storage ${STORAGE}"
+echo "                     ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${build_outerr}"
 # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
-export EESSI_REPOS_CFG_DIR_OVERRIDE=$(pwd)/cfg
+export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
 ./eessi_container.sh --access rw \
                      ${CONTAINER_OPT} \
                      ${HTTP_PROXY_OPT} \
@@ -142,6 +163,44 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=$(pwd)/cfg
                      --info \
                      --mode run \
                      ${REPOSITORY_OPT} \
-                     --save $(pwd)/previous_tmp \
+                     --save ${PWD}/previous_tmp \
                      --storage ${STORAGE} \
-                     ./install_software_layer.sh "$@" 2>&1 | tee -a ${run_outerr}
+                     ./install_software_layer.sh "$@" 2>&1 | tee -a ${build_outerr}
+
+# determine temporary directory to resume from
+BUILD_TMPDIR=$(grep 'RESUME_FROM_DIR' ${build_outerr} | sed -e "s/^RESUME_FROM_DIR //")
+
+tar_outerr=$(mktemp tar.outerr.XXXX)
+timestamp=$(date +%s)
+# to set EESSI_PILOT_VERSION we need to source init/eessi_defaults now
+source init/eessi_defaults
+export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp})
+
+# value of first parameter to create_tarball.sh - TMP_IN_CONTAINER - needs to be
+# synchronised with setting of TMP_IN_CONTAINER in eessi_container.sh
+# TODO should we make this a configurable parameter of eessi_container.sh using
+# /tmp as default?
+TMP_IN_CONTAINER=/tmp
+echo "Executing command to create tarball:"
+echo "./eessi_container.sh --access rw"
+echo "                     ${CONTAINER_OPT}"
+echo "                     ${HTTP_PROXY_OPT}"
+echo "                     ${HTTPS_PROXY_OPT}"
+echo "                     --info"
+echo "                     --mode run"
+echo "                     ${REPOSITORY_OPT}"
+echo "                     --resume ${BUILD_TMPDIR}"
+echo "                     --save ${PWD}/previous_tmp"
+echo "                     ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
+./eessi_container.sh --access rw \
+                     ${CONTAINER_OPT} \
+                     ${HTTP_PROXY_OPT} \
+                     ${HTTPS_PROXY_OPT} \
+                     --info \
+                     --mode run \
+                     ${REPOSITORY_OPT} \
+                     --resume ${BUILD_TMPDIR} \
+                     --save ${PWD}/previous_tmp \
+                     ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
+
+exit 0
diff --git a/eessi_container.sh b/eessi_container.sh
index 3a9fac2955..1eb630a096 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -46,9 +46,15 @@ RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11))
 CVMFS_VAR_LIB="var-lib-cvmfs"
 CVMFS_VAR_RUN="var-run-cvmfs"
 
+# directory for tmp used inside container
+export TMP_IN_CONTAINER=/tmp
+
 # repository cfg file, default name (default location: $PWD)
 #   can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE
-export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=.}/repos.cfg"
+export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}/repos.cfg"
+# other repository cfg files in directory, default location: $PWD
+#   can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE
+export EESSI_REPOS_CFG_DIR="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}"
 
 
 # 0. parse args
@@ -193,7 +199,7 @@ fi
 # 1. check if argument values are valid
 # (arg -a|--access) check if ACCESS is supported
 if [[ "${ACCESS}" != "ro" && "${ACCESS}" != "rw" ]]; then
-  fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}"
+    fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}"
 fi
 
 # TODO (arg -c|--container) check container (is it a file or URL & access those)
@@ -205,11 +211,14 @@ fi
 
 # (arg -m|--mode) check if MODE is known
 if [[ "${MODE}" != "shell" && "${MODE}" != "run" ]]; then
-  fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}"
+    fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}"
 fi
 
 # TODO (arg -r|--repository) check if repository is known
 # REPOSITORY_ERROR_EXITCODE
+if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI-pilot" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then
+    fatal_error "arg '--repository ${REPOSITORY}' requires a cfg file at '${EESSI_REPOS_CFG_FILE}'" "${REPOSITORY_ERROR_EXITCODE}"
+fi
 
 # TODO (arg -u|--resume) check if it exists, if user has read permission,
 #      if it contains data from a previous run
@@ -243,6 +252,7 @@ if [[ ! -z ${RESUME} && -d ${RESUME} ]]; then
   #   skip creating a new tmp directory, just set environment variables
   echo "Resuming from previous run using temporary storage at ${RESUME}"
   EESSI_HOST_STORAGE=${RESUME}
+  echo "RESUME_FROM_DIR ${EESSI_HOST_STORAGE}"
 else
   # we need a tmp location (and possibly init it with ${RESUME} if it was not
   #   a directory
@@ -319,7 +329,9 @@ fi
 
 # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined)
 BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs"
-BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp"
+# provide a '/tmp' inside the container
+BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}"
+
 [[ ${INFO} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}"
 
 # set up repository config (always create directory repos_cfg and populate it with info when
@@ -375,13 +387,22 @@ else
   # use information to set up dir ${EESSI_TMPDIR}/repos_cfg,
   #     define BIND mounts and override repo name and version
   # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/repos_cfg
-  if [[ ! -r ${config_bundle} ]]; then
-    fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE}
+  # if config_bundle is relative path (no '/' at start) prepend it with
+  # EESSI_REPOS_CFG_DIR
+  config_bundle_path=
+  if [[ ! "${config_bundle}" =~ ^/ ]]; then
+      config_bundle_path=${EESSI_REPOS_CFG_DIR}/${config_bundle}
+  else
+      config_bundle_path=${config_bundle}
+  fi
+
+  if [[ ! -r ${config_bundle_path} ]]; then
+    fatal_error "config bundle '${config_bundle_path}' is not readable" ${REPOSITORY_ERROR_EXITCODE}
   fi
 
   # only unpack config_bundle if we're not resuming from a previous run
   if [[ -z ${RESUME} ]]; then
-    tar xf ${config_bundle} -C ${EESSI_TMPDIR}/repos_cfg
+    tar xf ${config_bundle_path} -C ${EESSI_TMPDIR}/repos_cfg
   fi
 
   for src in "${!cfg_file_map[@]}"
@@ -430,12 +451,8 @@ if [[ "${ACCESS}" == "ro" ]]; then
 fi
 
 if [[ "${ACCESS}" == "rw" ]]; then
-  EESSI_CVMFS_OVERLAY_UPPER=/tmp/overlay-upper
-  EESSI_CVMFS_OVERLAY_WORK=/tmp/overlay-work
   mkdir -p ${EESSI_TMPDIR}/overlay-upper
   mkdir -p ${EESSI_TMPDIR}/overlay-work
-  [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER='${EESSI_CVMFS_OVERLAY_UPPER}'"
-  [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK='${EESSI_CVMFS_OVERLAY_WORK}'"
 
   # set environment variables for fuse mounts in Singularity container
   export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}"
@@ -444,8 +461,8 @@ if [[ "${ACCESS}" == "rw" ]]; then
 
   EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs"
   EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}"
-  EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=/tmp/overlay-upper"
-  EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=/tmp/overlay-work"
+  EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper"
+  EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work"
   EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}"
   export EESSI_PILOT_WRITABLE_OVERLAY
 
@@ -470,11 +487,13 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
 fi
 
+# if INFO is set to 1 (arg --info), add argument '-q'
+RUN_QUIET=${INFO:--q}
 echo "Launching container with command (next line):"
-echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
+echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
 # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level
 #      provided to the script
-singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"
+singularity ${RUN_QUIET} ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"
 
 # 6. save tmp if requested (arg -s|--save)
 if [[ ! -z ${SAVE} ]]; then
@@ -493,6 +512,7 @@ if [[ ! -z ${SAVE} ]]; then
   fi
   tar cf ${TGZ} -C ${EESSI_TMPDIR} .
   echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')"
+  echo "RESUME_FROM_TGZ ${TGZ}"
 fi
 
 # TODO clean up tmp by default? only retain if another option provided (--retain-tmp)
diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh
index e4850d677d..c70077bf15 100755
--- a/run_in_compat_layer_env.sh
+++ b/run_in_compat_layer_env.sh
@@ -3,7 +3,6 @@
 base_dir=$(dirname $(realpath $0))
 source ${base_dir}/init/eessi_defaults
 
-BUILD_CONTAINER="docker://ghcr.io/eessi/build-node:debian10"
 if [ -z $EESSI_PILOT_VERSION ]; then
     echo "ERROR: \$EESSI_PILOT_VERSION must be set!" >&2
     exit 1
@@ -18,6 +17,18 @@ INPUT=$(echo "$@")
 if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     INPUT="export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}; ${INPUT}"
 fi
+if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then
+    INPUT="export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}; ${INPUT}"
+fi
+if [ ! -z ${EESSI_PILOT_VERSION_OVERRIDE} ]; then
+    INPUT="export EESSI_PILOT_VERSION_OVERRIDE=${EESSI_PILOT_VERSION_OVERRIDE}; ${INPUT}"
+fi
+if [ ! -z ${http_proxy} ]; then
+    INPUT="export http_proxy=${http_proxy}; ${INPUT}"
+fi
+if [ ! -z ${https_proxy} ]; then
+    INPUT="export https_proxy=${https_proxy}; ${INPUT}"
+fi
 
-echo "Running '${INPUT}' in EESSI ${EESSI_PILOT_VERSION} compatibility layer environment..."
+echo "Running '${INPUT}' in EESSI (${EESSI_CVMFS_REPO}) ${EESSI_PILOT_VERSION} compatibility layer environment..."
 ${EESSI_COMPAT_LAYER_DIR}/startprefix <<< "${INPUT}"

From 3a6fe0e4ec0b10387bfd43cf346c3f052b4b85f3 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Wed, 8 Feb 2023 08:49:37 +0100
Subject: [PATCH 052/157] update location of utils.sh, uniform way to determine
 'TOPDIR'

---
 eessi_container.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 57b003368b..29a2ea4565 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -23,13 +23,13 @@
 # 6. run container
 
 # -. initial settings & exit codes
-base_dir=$(dirname $(realpath $0))
+TOPDIR=$(dirname $(realpath $0))
 
-source ${base_dir}/utils.sh
-source ${base_dir}/cfg_files.sh
+source ${TOPDIR}/scripts/utils.sh
+source ${TOPDIR}/cfg_files.sh
 
 # exit codes: bitwise shift codes to allow for combination of exit codes
-# ANY_ERROR_EXITCODE is sourced from ${base_dir}/utils.sh
+# ANY_ERROR_EXITCODE is sourced from ${TOPDIR}/scripts/utils.sh
 CMDLINE_ARG_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 1))
 ACCESS_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 2))
 CONTAINER_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 3))

From 756b1b62407c250dfe8b7dcf9d87511375c7580c Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Wed, 8 Feb 2023 08:53:54 +0100
Subject: [PATCH 053/157] bump default container version

---
 eessi_container.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 29a2ea4565..2b2c0716ca 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -61,7 +61,7 @@ display_help() {
   echo " OPTIONS:"
   echo "  -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]"
   echo "  -c | --container IMG  - image file or URL defining the container to use"
-  echo "                          [default: docker://ghcr.io/eessi/build-node:debian10]"
+  echo "                          [default: docker://ghcr.io/eessi/build-node:debian11]"
   echo "  -h | --help           - display this usage information [default: false]"
   echo "  -g | --storage DIR    - directory space on host machine (used for"
   echo "                          temporary data) [default: 1. TMPDIR, 2. /tmp]"
@@ -95,7 +95,7 @@ display_help() {
 
 # set defaults for command line arguments
 ACCESS="ro"
-CONTAINER="docker://ghcr.io/eessi/build-node:debian10"
+CONTAINER="docker://ghcr.io/eessi/build-node:debian11"
 DRY_RUN=0
 INFO=0
 STORAGE=

From 4c435ab94185c8ab1cb64eebc97ffa96d47e83fb Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Wed, 8 Feb 2023 08:57:00 +0100
Subject: [PATCH 054/157] clarifying default value for repository arg

---
 eessi_container.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 2b2c0716ca..b6eb3894ef 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -68,7 +68,8 @@ display_help() {
   echo "  -m | --mode MODE      - with MODE==shell (launch interactive shell) or"
   echo "                          MODE==run (run a script) [default: shell]"
   echo "  -r | --repository CFG - configuration file or identifier defining the"
-  echo "                          repository to use [default: EESSI-pilot]"
+  echo "                          repository to use [default: EESSI-pilot via"
+  echo "                          container configuration]"
   echo "  -u | --resume DIR/TGZ - resume a previous run from a directory or tarball,"
   echo "                          where DIR points to a previously used tmp directory"
   echo "                          (check for output 'Using DIR as tmp ...' of a previous"

From 990ebdae4dd77e396add2b2237efa1f6a70708d6 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Feb 2023 08:57:54 +0100
Subject: [PATCH 055/157] move commands to install Apptainer in GitHub Actions
 Ubuntu VM to a script

---
 .github/workflows/tests_scripts.yml | 11 +----------
 install_apptainer_ubuntu.sh         | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 10 deletions(-)
 create mode 100755 install_apptainer_ubuntu.sh

diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml
index 9c4975c381..ca23a15c40 100644
--- a/.github/workflows/tests_scripts.yml
+++ b/.github/workflows/tests_scripts.yml
@@ -27,18 +27,9 @@ jobs:
     - name: checkout
       uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
-    # see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181
     - name: install Apptainer
       run: |
-        sudo apt-get install alien
-        alien --version
-        apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g')
-        curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/$apptainer_rpm
-        sudo alien -d $apptainer_rpm
-        sudo apt install ./apptainer*.deb
-        apptainer --version
-        # also check whether 'singularity' command is still provided by Apptainer installation
-        singularity --version
+        ./install_apptainer_ubuntu.sh
 
     - name: test install_software_layer.sh script
       run: |
diff --git a/install_apptainer_ubuntu.sh b/install_apptainer_ubuntu.sh
new file mode 100755
index 0000000000..c35c34cda6
--- /dev/null
+++ b/install_apptainer_ubuntu.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+# see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181
+sudo apt-get install alien
+alien --version
+apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g')
+curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/${apptainer_rpm}
+sudo alien -d ${apptainer_rpm}
+sudo apt install ./apptainer*.deb
+apptainer --version
+# also check whether 'singularity' command is still provided by Apptainer installation
+singularity --version

From 3d143d853dbecffb746d5ab3610d188d4ca12bda Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Feb 2023 08:58:00 +0100
Subject: [PATCH 056/157] add GitHub Actions workflow for testing
 eessi_container.sh script

---
 .../workflows/test_eessi_container_script.yml | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 .github/workflows/test_eessi_container_script.yml

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
new file mode 100644
index 0000000000..c02cd0dacf
--- /dev/null
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -0,0 +1,28 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Tests for eessi_container.sh script
+on: [push, pull_request, workflow_dispatch]
+permissions:
+  contents: read # to fetch code (actions/checkout)
+jobs:
+  eessi_container_script:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        SCRIPT_ARGS:
+        - "--help"
+        - "--info"
+    steps:
+        - name: Check out software-layer repository
+          uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+
+        - name: Mount EESSI CernVM-FS pilot repository
+          uses: eessi/github-action-eessi@v2
+
+        - name: install Apptainer
+          run: |
+            ./install_apptainer_ubuntu.sh
+
+        - name: Test eessi_container.sh script
+          run: |
+              ./eessi_container.sh ${{matrix.SCRIPT_ARGS}}

From 5fe467a04cb188f47f2b52c0e7159ef64a38ab58 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Feb 2023 09:13:03 +0100
Subject: [PATCH 057/157] stop script as soon as any command does non-zero exit
 + undefined env vars are errors

---
 eessi_container.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/eessi_container.sh b/eessi_container.sh
index b6eb3894ef..c09ea75ebf 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -12,6 +12,10 @@
 # license: GPLv2
 #
 
+# -e: stop script as soon as any command has non-zero exit code
+# -u: treat usage of undefined variables as errors
+set -e -u
+
 # script overview
 # -. initial settings & exit codes
 # 0. parse args

From 57b831526fadf55ca1f886fd92efc11d75aa93f4 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Feb 2023 09:24:42 +0100
Subject: [PATCH 058/157] fix typo

---
 eessi_container.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index c09ea75ebf..283c41ddc0 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -318,7 +318,7 @@ BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp"
 mkdir -p ${EESSI_TMPDIR}/cfg
 if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then
   # need to source defaults as late as possible (see other sourcing below)
-  source ${base_dir}/init/eessi_defaults
+  source ${TOPDIR}/init/eessi_defaults
 
   # strip "/cvmfs/" from default setting
   repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//}
@@ -383,7 +383,7 @@ else
   export EESSI_PILOT_VERSION_OVERRIDE=${repo_version}
   export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}"
   # need to source defaults as late as possible (after *_OVERRIDEs)
-  source ${base_dir}/init/eessi_defaults
+  source ${TOPDIR}/init/eessi_defaults
 fi
 
 

From fba368d19686c1f2edaeb524d0a78f86673b453e Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Feb 2023 09:25:20 +0100
Subject: [PATCH 059/157] comment out 'set -e -u' for now

---
 eessi_container.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 283c41ddc0..a63eb3ad44 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -14,7 +14,8 @@
 
 # -e: stop script as soon as any command has non-zero exit code
 # -u: treat usage of undefined variables as errors
-set -e -u
+# FIXME commented out because it's OK (?) if some environment variables are not set (like $SINGULARITY_HOME)
+# set -e -u
 
 # script overview
 # -. initial settings & exit codes

From 09feff74d11d4cf14393cde33abd0672eb9f3bcf Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Feb 2023 10:27:05 +0100
Subject: [PATCH 060/157] rename --info to --verbose

---
 .../workflows/test_eessi_container_script.yml |  2 +-
 eessi_container.sh                            | 34 +++++++++----------
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index c02cd0dacf..a940a86924 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -11,7 +11,7 @@ jobs:
       matrix:
         SCRIPT_ARGS:
         - "--help"
-        - "--info"
+        - "--verbose"
     steps:
         - name: Check out software-layer repository
           uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
diff --git a/eessi_container.sh b/eessi_container.sh
index a63eb3ad44..166f5ff2a4 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -92,7 +92,7 @@ display_help() {
   echo " FEATURES/OPTIONS to be implemented:"
   echo "  -d | --dry-run           -  run script except for executing the container,"
   echo "                              print information about setup [default: false]"
-  echo "  -i | --info              -  display configured repositories [default: false]"
+  echo "  -v | --verbose           -  display more information [default: false]"
   #echo "  -x | --http-proxy URL    -  provides URL for the env variable http_proxy"
   #echo "                              [default: not set]"
   #echo "  -y | --https-proxy URL   -  provides URL for the env variable https_proxy"
@@ -103,7 +103,7 @@ display_help() {
 ACCESS="ro"
 CONTAINER="docker://ghcr.io/eessi/build-node:debian11"
 DRY_RUN=0
-INFO=0
+VERBOSE=0
 STORAGE=
 MODE="shell"
 REPOSITORY="EESSI-pilot"
@@ -137,10 +137,6 @@ while [[ $# -gt 0 ]]; do
       display_help
       exit 0
       ;;
-    -i|--info)
-      INFO=1
-      shift 1
-      ;;
     -m|--mode)
       MODE="$2"
       shift 2
@@ -157,6 +153,10 @@ while [[ $# -gt 0 ]]; do
       RESUME="$2"
       shift 2
       ;;
+    -v|--verbose)
+      VERBOSE=1
+      shift 1
+      ;;
 #    -x|--http-proxy)
 #      HTTP_PROXY="$2"
 #      export http_proxy=${HTTP_PROXY}
@@ -261,7 +261,7 @@ else
     # mktemp falls back to using /tmp if TMPDIR is empty
     # TODO check if /tmp is writable, large enough and usable (different
     #      features for ro-access and rw-access)
-    [[ ${INFO} -eq 1 ]] && echo "skipping sanity checks for /tmp"
+    [[ ${VERBOSE} -eq 1 ]] && echo "skipping sanity checks for /tmp"
   fi
   EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX)
   echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)."
@@ -287,32 +287,32 @@ fi
 # tmp dir for EESSI
 EESSI_TMPDIR=${EESSI_HOST_STORAGE}
 mkdir -p ${EESSI_TMPDIR}
-[[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}"
+[[ ${VERBOSE} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}"
 
 # configure Singularity
 export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache
 mkdir -p ${SINGULARITY_CACHEDIR}
-[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}"
+[[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}"
 
 # set env vars and create directories for CernVM-FS
 EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB}
 EESSI_CVMFS_VAR_RUN=${EESSI_TMPDIR}/${CVMFS_VAR_RUN}
 mkdir -p ${EESSI_CVMFS_VAR_LIB}
 mkdir -p ${EESSI_CVMFS_VAR_RUN}
-[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_LIB=${EESSI_CVMFS_VAR_LIB}"
-[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_RUN=${EESSI_CVMFS_VAR_RUN}"
+[[ ${VERBOSE} -eq 1 ]] && echo "EESSI_CVMFS_VAR_LIB=${EESSI_CVMFS_VAR_LIB}"
+[[ ${VERBOSE} -eq 1 ]] && echo "EESSI_CVMFS_VAR_RUN=${EESSI_CVMFS_VAR_RUN}"
 
 # allow that SINGULARITY_HOME is defined before script is run
 if [[ -z ${SINGULARITY_HOME} ]]; then
   export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}"
   mkdir -p ${EESSI_TMPDIR}/home
-  [[ ${INFO} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}"
+  [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}"
 fi
 
 # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined)
 BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs"
 BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp"
-[[ ${INFO} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}"
+[[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}"
 
 # set up repository config (always create cfg dir and populate it with info when
 # arg -r|--repository is used)
@@ -362,7 +362,7 @@ else
 
   # convert config_map into associative array cfg_file_map
   cfg_init_file_map "${config_map}"
-  [[ ${INFO} -eq 1 ]] && cfg_print_map
+  [[ ${VERBOSE} -eq 1 ]] && cfg_print_map
 
   # TODO use information to set up dir ${EESSI_TMPDIR}/cfg,
   #      define BIND mounts and override repo name and version
@@ -403,8 +403,8 @@ if [[ "${ACCESS}" == "rw" ]]; then
   EESSI_CVMFS_OVERLAY_WORK=/tmp/overlay-work
   mkdir -p ${EESSI_TMPDIR}/overlay-upper
   mkdir -p ${EESSI_TMPDIR}/overlay-work
-  [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}"
-  [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}"
+  [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}"
+  [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}"
 
   # set environment variables for fuse mounts in Singularity container
   export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}"
@@ -433,7 +433,7 @@ if [[ -z ${SINGULARITY_BIND} ]]; then
 else
     export SINGULARITY_BIND="${SINGULARITY_BIND},${BIND_PATHS}"
 fi
-[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}"
+[[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}"
 
 echo "Launching container with command (next line):"
 echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}"

From fe83f85fe93bdb492946355ce611fcc96906cf8f Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Feb 2023 11:07:58 +0100
Subject: [PATCH 061/157] pin eessi/github-action-eessi by hash

---
 .github/workflows/test_eessi_container_script.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index a940a86924..fc4cd01f48 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -17,7 +17,7 @@ jobs:
           uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
         - name: Mount EESSI CernVM-FS pilot repository
-          uses: eessi/github-action-eessi@v2
+          uses: eessi/github-action-eessi@58b50fd2eead2162c2b9ac258d4fb60cc9f30503 # v2.0.13
 
         - name: install Apptainer
           run: |

From 125bfa25d30e849e783bacb68c16200bea75dd7e Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Feb 2023 11:23:38 +0100
Subject: [PATCH 062/157] use exit code of container command in
 eessi_container.sh

---
 eessi_container.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/eessi_container.sh b/eessi_container.sh
index 166f5ff2a4..81ad7b4b7a 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -440,6 +440,7 @@ echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_
 # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level
 #      provided to the script
 singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}
+exit_code=$?
 
 # 7. save tmp if requested (arg -s|--save)
 if [[ ! -z ${SAVE} ]]; then
@@ -461,3 +462,6 @@ if [[ ! -z ${SAVE} ]]; then
 fi
 
 # TODO clean up tmp by default? only retain if another option provided (--retain-tmp)
+
+# use exit code of container command
+exit ${exit_code}

From be3e39909797986845f2be4274ee5c4375c2ae7b Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Wed, 8 Feb 2023 13:01:21 +0100
Subject: [PATCH 063/157] replace RUN_SCRIPT_AND_ARGS with

---
 eessi_container.sh | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 166f5ff2a4..4d61677e46 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -111,7 +111,6 @@ RESUME=
 SAVE=
 #HTTP_PROXY=
 #HTTPS_PROXY=
-RUN_SCRIPT_AND_ARGS=
 
 POSITIONAL_ARGS=()
 
@@ -220,8 +219,6 @@ fi
 if [[ "${MODE}" == "run" ]]; then
   if [[ $# -eq 0 ]]; then
     fatal_error "no command specified to run?!" "${RUN_SCRIPT_MISSING_EXITCODE}"
-  else
-    RUN_SCRIPT_AND_ARGS=$@
   fi
 fi
 
@@ -436,10 +433,10 @@ fi
 [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}"
 
 echo "Launching container with command (next line):"
-echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}"
+echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
 # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level
 #      provided to the script
-singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}
+singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"
 
 # 7. save tmp if requested (arg -s|--save)
 if [[ ! -z ${SAVE} ]]; then

From 27cbdf3e0c3507925148945e41dc0683772489c6 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Feb 2023 15:52:49 +0100
Subject: [PATCH 064/157] add checks for various use cases of
 eessi_container.sh script (+ no need to natively mount EESSI, since that's
 done in the container)

---
 .../workflows/test_eessi_container_script.yml | 86 +++++++++++++++++--
 1 file changed, 79 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index fc4cd01f48..d1b6b1cb74 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -9,20 +9,92 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        SCRIPT_ARGS:
-        - "--help"
-        - "--verbose"
+        SCRIPT_TEST:
+        - help
+        - run
+        - shell
+        - container
+        - readwrite
+        - resume
+        - save
     steps:
         - name: Check out software-layer repository
           uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
-        - name: Mount EESSI CernVM-FS pilot repository
-          uses: eessi/github-action-eessi@58b50fd2eead2162c2b9ac258d4fb60cc9f30503 # v2.0.13
-
         - name: install Apptainer
           run: |
             ./install_apptainer_ubuntu.sh
 
         - name: Test eessi_container.sh script
           run: |
-              ./eessi_container.sh ${{matrix.SCRIPT_ARGS}}
+              test_cmd="cat /etc/os-release"
+              out_pattern="Debian GNU/Linux 11"
+
+              if [[ ${{matrix.SCRIPT_TEST}} == 'help' ]]; then
+                ./eessi_container.sh --help
+
+              # test use of --mode run
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'run' ]]; then
+                outfile=out_run.txt
+                echo "${test_cmd}" > test_script.sh
+                chmod u+x test_script.sh
+                export SINGULARITY_BIND="$PWD:/test"
+                ./eessi_container.sh --mode run /test/test_script.sh | tee ${outfile}
+                grep "${out_pattern}" ${outfile}
+
+              # test use of --mode shell
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'shell' ]]; then
+                outfile=out_shell.txt
+                ./eessi_container.sh --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                grep "${out_pattern}" ${outfile}
+
+              # test use of --container option, using a totally different container;
+              # cfr. https://github.com/easybuilders/easybuild-containers
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'container' ]]; then
+                outfile=out_container.txt
+                container="docker://ghcr.io/eessi/build-node:debian10"
+                ./eessi_container.sh --container ${container} --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                grep "${out_pattern}" ${outfile}
+
+              # test use of '--access rw' to get write access in container
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'readwrite' ]]; then
+                outfile=out_readwrite.txt
+                fn="test_${RANDOM}.txt"
+                echo "touch /cvmfs/pilot.eessi-hpc.org/${fn}" > test_script.sh
+                chmod u+x test_script.sh
+                export SINGULARITY_BIND="$PWD:/test"
+                ./eessi_container.sh --access rw --mode run /test/test_script.sh > ${outfile}
+
+                tmpdir=$(grep "--resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
+                # note: must use '--access rw' again here, since touched file is in overlay upper dir
+                ./eessi_container.sh --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
+                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$"> $outfile
+
+              # test use of --resume
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then
+                outfile=out_resume.txt
+                ./eessi_container.sh --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                rm -f ${outfile}
+
+                tmpdir=$(grep "--resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
+                ./eessi_container.sh --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile}
+                grep "Resuming from previous run using temporary storage at ${tmpdir}" ${outfile}
+                grep "${out_pattern}" ${outfile}
+
+              # test use of --save (+ --resume)
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'save' ]]; then
+                outfile=out_save.txt
+                fn="test_${RANDOM}.txt"
+                test_cmd="touch /cvmfs/pilot.eessi-hpc.org/${fn}"
+                ./eessi_container.sh --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                rm -f ${outfile}
+
+                ./eessi_container.sh --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
+                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$"> $outfile
+
+                tar tfv test-save.tar | grep "overlay-upper/${fn}"
+
+              else
+                  echo "Unknown test case: ${{matrix.SCRIPT_TEST}}" >&2
+                  exit 1
+              fi

From b34afca53a24c72ba69f23956e1d55c7a5cc6585 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Wed, 8 Feb 2023 16:18:08 +0100
Subject: [PATCH 065/157] commented dry-run option, moved verbose option in
 help msg

---
 eessi_container.sh | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index e0f4ead1af..96a9900a5e 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -86,13 +86,13 @@ display_help() {
   echo "                          when a directory is provided, the format of the"
   echo "                          tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz"
   echo "                          [default: not set]"
+  echo "  -v | --verbose        - display more information [default: false]"
   echo
   echo " If value for --mode is 'run', the SCRIPT provided is executed."
-  echo
-  echo " FEATURES/OPTIONS to be implemented:"
-  echo "  -d | --dry-run           -  run script except for executing the container,"
-  echo "                              print information about setup [default: false]"
-  echo "  -v | --verbose           -  display more information [default: false]"
+  #echo
+  #echo " FEATURES/OPTIONS to be implemented:"
+  #echo "  -d | --dry-run           -  run script except for executing the container,"
+  #echo "                              print information about setup [default: false]"
   #echo "  -x | --http-proxy URL    -  provides URL for the env variable http_proxy"
   #echo "                              [default: not set]"
   #echo "  -y | --https-proxy URL   -  provides URL for the env variable https_proxy"
@@ -102,7 +102,7 @@ display_help() {
 # set defaults for command line arguments
 ACCESS="ro"
 CONTAINER="docker://ghcr.io/eessi/build-node:debian11"
-DRY_RUN=0
+#DRY_RUN=0
 VERBOSE=0
 STORAGE=
 MODE="shell"
@@ -124,10 +124,10 @@ while [[ $# -gt 0 ]]; do
       CONTAINER="$2"
       shift 2
       ;;
-    -d|--dry-run)
-      DRY_RUN=1
-      shift 1
-      ;;
+#    -d|--dry-run)
+#      DRY_RUN=1
+#      shift 1
+#      ;;
     -g|--storage)
       STORAGE="$2"
       shift 2

From 7d3818d539111186f7d33dbf1b3a34813cd4e58b Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Feb 2023 15:52:49 +0100
Subject: [PATCH 066/157] add checks for various use cases of
 eessi_container.sh script (+ no need to natively mount EESSI, since that's
 done in the container)

---
 .../workflows/test_eessi_container_script.yml | 86 +++++++++++++++++--
 1 file changed, 79 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index fc4cd01f48..362bd3893c 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -9,20 +9,92 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        SCRIPT_ARGS:
-        - "--help"
-        - "--verbose"
+        SCRIPT_TEST:
+        - help
+        - run
+        - shell
+        - container
+        - readwrite
+        - resume
+        - save
     steps:
         - name: Check out software-layer repository
           uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
-        - name: Mount EESSI CernVM-FS pilot repository
-          uses: eessi/github-action-eessi@58b50fd2eead2162c2b9ac258d4fb60cc9f30503 # v2.0.13
-
         - name: install Apptainer
           run: |
             ./install_apptainer_ubuntu.sh
 
         - name: Test eessi_container.sh script
           run: |
-              ./eessi_container.sh ${{matrix.SCRIPT_ARGS}}
+              test_cmd="cat /etc/os-release"
+              out_pattern="Debian GNU/Linux 11"
+
+              if [[ ${{matrix.SCRIPT_TEST}} == 'help' ]]; then
+                ./eessi_container.sh --help
+
+              # test use of --mode run
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'run' ]]; then
+                outfile=out_run.txt
+                echo "${test_cmd}" > test_script.sh
+                chmod u+x test_script.sh
+                export SINGULARITY_BIND="$PWD:/test"
+                ./eessi_container.sh --mode run /test/test_script.sh | tee ${outfile}
+                grep "${out_pattern}" ${outfile}
+
+              # test use of --mode shell
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'shell' ]]; then
+                outfile=out_shell.txt
+                ./eessi_container.sh --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                grep "${out_pattern}" ${outfile}
+
+              # test use of --container option, using a totally different container;
+              # cfr. https://github.com/easybuilders/easybuild-containers
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'container' ]]; then
+                outfile=out_container.txt
+                container="docker://ghcr.io/eessi/build-node:debian10"
+                ./eessi_container.sh --container ${container} --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                grep "Debian GNU/Linux 10" ${outfile}
+
+              # test use of '--access rw' to get write access in container
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'readwrite' ]]; then
+                outfile=out_readwrite.txt
+                fn="test_${RANDOM}.txt"
+                echo "touch /cvmfs/pilot.eessi-hpc.org/${fn}" > test_script.sh
+                chmod u+x test_script.sh
+                export SINGULARITY_BIND="$PWD:/test"
+                ./eessi_container.sh --access rw --mode run /test/test_script.sh > ${outfile}
+
+                tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
+                # note: must use '--access rw' again here, since touched file is in overlay upper dir
+                ./eessi_container.sh --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
+                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$"> $outfile
+
+              # test use of --resume
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then
+                outfile=out_resume.txt
+                ./eessi_container.sh --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                rm -f ${outfile}
+
+                tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
+                ./eessi_container.sh --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile}
+                grep "Resuming from previous run using temporary storage at ${tmpdir}" ${outfile}
+                grep "${out_pattern}" ${outfile}
+
+              # test use of --save (+ --resume)
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'save' ]]; then
+                outfile=out_save.txt
+                fn="test_${RANDOM}.txt"
+                test_cmd="touch /cvmfs/pilot.eessi-hpc.org/${fn}"
+                ./eessi_container.sh --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                rm -f ${outfile}
+
+                ./eessi_container.sh --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
+                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$"> $outfile
+
+                tar tfv test-save.tar | grep "overlay-upper/${fn}"
+
+              else
+                  echo "Unknown test case: ${{matrix.SCRIPT_TEST}}" >&2
+                  exit 1
+              fi

From 602aa68068b275b56b181af9bbcbf12f8395eedb Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Feb 2023 16:28:52 +0100
Subject: [PATCH 067/157] fix check for './eessi_container.sh --resume'

---
 .github/workflows/test_eessi_container_script.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index 362bd3893c..4d1c3bf898 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -73,11 +73,13 @@ jobs:
               # test use of --resume
               elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then
                 outfile=out_resume.txt
-                ./eessi_container.sh --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
-                rm -f ${outfile}
+                ./eessi_container.sh --mode shell <<< "${test_cmd}" > ${outfile}
 
                 tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
+                rm -f ${outfile}
+
                 ./eessi_container.sh --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile}
+                cat ${outfile}
                 grep "Resuming from previous run using temporary storage at ${tmpdir}" ${outfile}
                 grep "${out_pattern}" ${outfile}
 

From adcc655f99c65a609bc1cf21f1fe9d6fc89a8c7c Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Wed, 8 Feb 2023 17:01:26 +0100
Subject: [PATCH 068/157] fix small typos in grep commands

---
 .github/workflows/test_eessi_container_script.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index 362bd3893c..3974b42506 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -68,7 +68,7 @@ jobs:
                 tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
                 # note: must use '--access rw' again here, since touched file is in overlay upper dir
                 ./eessi_container.sh --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
-                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$"> $outfile
+                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile
 
               # test use of --resume
               elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then
@@ -90,7 +90,7 @@ jobs:
                 rm -f ${outfile}
 
                 ./eessi_container.sh --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
-                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$"> $outfile
+                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile
 
                 tar tfv test-save.tar | grep "overlay-upper/${fn}"
 

From 7e470233684d456830f9cb7559992a7140845a57 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 10 Feb 2023 09:11:46 +0100
Subject: [PATCH 069/157] enable --verbose + add step to collect test env info

---
 .../workflows/test_eessi_container_script.yml | 23 +++++++++++--------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index c8c2aef47e..0920678c24 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -25,6 +25,11 @@ jobs:
           run: |
             ./install_apptainer_ubuntu.sh
 
+        - name: Collect info on test environment
+          run: |
+              mount
+              df -h
+
         - name: Test eessi_container.sh script
           run: |
               test_cmd="cat /etc/os-release"
@@ -39,13 +44,13 @@ jobs:
                 echo "${test_cmd}" > test_script.sh
                 chmod u+x test_script.sh
                 export SINGULARITY_BIND="$PWD:/test"
-                ./eessi_container.sh --mode run /test/test_script.sh | tee ${outfile}
+                ./eessi_container.sh --verbose --mode run /test/test_script.sh | tee ${outfile}
                 grep "${out_pattern}" ${outfile}
 
               # test use of --mode shell
               elif [[ ${{matrix.SCRIPT_TEST}} == 'shell' ]]; then
                 outfile=out_shell.txt
-                ./eessi_container.sh --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                ./eessi_container.sh --verbose --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
                 grep "${out_pattern}" ${outfile}
 
               # test use of --container option, using a totally different container;
@@ -53,7 +58,7 @@ jobs:
               elif [[ ${{matrix.SCRIPT_TEST}} == 'container' ]]; then
                 outfile=out_container.txt
                 container="docker://ghcr.io/eessi/build-node:debian10"
-                ./eessi_container.sh --container ${container} --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                ./eessi_container.sh --verbose --container ${container} --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
                 grep "Debian GNU/Linux 10" ${outfile}
 
               # test use of '--access rw' to get write access in container
@@ -63,22 +68,22 @@ jobs:
                 echo "touch /cvmfs/pilot.eessi-hpc.org/${fn}" > test_script.sh
                 chmod u+x test_script.sh
                 export SINGULARITY_BIND="$PWD:/test"
-                ./eessi_container.sh --access rw --mode run /test/test_script.sh > ${outfile}
+                ./eessi_container.sh --verbose --access rw --mode run /test/test_script.sh > ${outfile}
 
                 tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
                 # note: must use '--access rw' again here, since touched file is in overlay upper dir
-                ./eessi_container.sh --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
+                ./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
                 grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile
 
               # test use of --resume
               elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then
                 outfile=out_resume.txt
-                ./eessi_container.sh --mode shell <<< "${test_cmd}" > ${outfile}
+                ./eessi_container.sh --verbose --mode shell <<< "${test_cmd}" > ${outfile}
 
                 tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
                 rm -f ${outfile}
 
-                ./eessi_container.sh --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile}
+                ./eessi_container.sh --verbose --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile}
                 cat ${outfile}
                 grep "Resuming from previous run using temporary storage at ${tmpdir}" ${outfile}
                 grep "${out_pattern}" ${outfile}
@@ -88,10 +93,10 @@ jobs:
                 outfile=out_save.txt
                 fn="test_${RANDOM}.txt"
                 test_cmd="touch /cvmfs/pilot.eessi-hpc.org/${fn}"
-                ./eessi_container.sh --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                ./eessi_container.sh --verbose --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile}
                 rm -f ${outfile}
 
-                ./eessi_container.sh --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
+                ./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
                 grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile
 
                 tar tfv test-save.tar | grep "overlay-upper/${fn}"

From a1bb7df72857be55e7e626de09634d1015cb45c0 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 10 Feb 2023 09:35:56 +0100
Subject: [PATCH 070/157] disable broken tests for eessi_container.sh script
 for now

---
 .github/workflows/test_eessi_container_script.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index 0920678c24..7678aedc64 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -14,9 +14,10 @@ jobs:
         - run
         - shell
         - container
-        - readwrite
         - resume
-        - save
+        # FIXME disabled because '--access rw' is not working in CI environment
+        #- readwrite
+        #- save
     steps:
         - name: Check out software-layer repository
           uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0

From b1a36355de66fa9a9557edbef79508f88c763cbb Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Tue, 24 Jan 2023 21:59:41 +0100
Subject: [PATCH 071/157] added a function to obtain full path to a tool

---
 scripts/utils.sh | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/scripts/utils.sh b/scripts/utils.sh
index 5d8455bb68..cd7f758a0d 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -31,3 +31,35 @@ function check_exit_code {
         fatal_error "${fail_msg}"
     fi
 }
+
+function get_path_for_tool {
+    tool_name=$1
+    tool_envvar_name=$2
+
+    which_out=$(which ${tool_name} 2>&1)
+    exit_code=$?
+    if [[ ${exit_code} -eq 0 ]]; then
+        echo "INFO: found tool ${tool_name} in PATH (${which_out})" >&2
+        echo "${which_out}"
+        return 0
+    fi
+    if [[ -z "${tool_envvar_name}" ]]; then
+        msg="no env var holding the full path to tool '${tool_name}' provided"
+        echo "${msg}" >&2
+        return 1
+    else
+        tool_envvar_value=${!tool_envvar_name}
+        if [[ -x "${tool_envvar_value}" ]]; then
+            msg="INFO: found tool ${tool_envvar_value} via env var ${tool_envvar_name}"
+            echo "${msg}" >&2
+            echo "${tool_envvar_value}"
+            return 0
+        else
+            msg="ERROR: tool '${tool_name}' not in PATH\n"
+            msg+="ERROR: tool '${tool_envvar_value}' via '${tool_envvar_name}' not in PATH"
+            echo "${msg}" >&2
+            echo ""
+            return 2
+        fi
+    fi
+}

From 27bc2035b00f59442fd46bcdbd6e104c09944f18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= <thomas.roblitz@uib.no>
Date: Wed, 25 Jan 2023 19:37:55 +0100
Subject: [PATCH 072/157] added function to determine IPv4 address

---
 scripts/utils.sh | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/scripts/utils.sh b/scripts/utils.sh
index cd7f758a0d..db533adc12 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -63,3 +63,12 @@ function get_path_for_tool {
         fi
     fi
 }
+
+function get_ipv4_address {
+    hname=$1
+    hipv4=$(grep ${hname} /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1)
+    # TODO try other methods if the one above does not work --> tool that verifies
+    #      what method can be used?
+    echo "${hipv4}"
+    return 0
+}

From bfe6d36bf056c9e4e9719fa3a8110899fb125dc2 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 18 Feb 2023 20:44:16 +0100
Subject: [PATCH 073/157] various updates for bot/build.sh and
 eessi_container.sh

cherry-picked via commit b5bf008e4136b7877e06ca0602b6fd6aa49128c0
ONLY picked chages to eessi_container.sh
---
 eessi_container.sh | 103 ++++++++++++++++++++++++---------------------
 1 file changed, 55 insertions(+), 48 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 96a9900a5e..3045fd5a89 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -43,8 +43,8 @@ MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5))
 REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6))
 RESUME_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7))
 SAVE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 8))
-#HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9))
-#HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10))
+HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9))
+HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10))
 RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11))
 
 # CernVM-FS settings
@@ -64,39 +64,39 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}"
 display_help() {
   echo "usage: $0 [OPTIONS] [SCRIPT]"
   echo " OPTIONS:"
-  echo "  -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]"
-  echo "  -c | --container IMG  - image file or URL defining the container to use"
-  echo "                          [default: docker://ghcr.io/eessi/build-node:debian11]"
-  echo "  -h | --help           - display this usage information [default: false]"
-  echo "  -g | --storage DIR    - directory space on host machine (used for"
-  echo "                          temporary data) [default: 1. TMPDIR, 2. /tmp]"
-  echo "  -m | --mode MODE      - with MODE==shell (launch interactive shell) or"
-  echo "                          MODE==run (run a script) [default: shell]"
-  echo "  -r | --repository CFG - configuration file or identifier defining the"
-  echo "                          repository to use [default: EESSI-pilot via"
+  echo "  -a | --access {ro,rw}  - ro (read-only), rw (read & write) [default: ro]"
+  echo "  -c | --container IMG   - image file or URL defining the container to use"
+  echo "                           [default: docker://ghcr.io/eessi/build-node:debian11]"
+  echo "  -h | --help            - display this usage information [default: false]"
+  echo "  -g | --storage DIR     - directory space on host machine (used for"
+  echo "                           temporary data) [default: 1. TMPDIR, 2. /tmp]"
+  echo "  -m | --mode MODE       - with MODE==shell (launch interactive shell) or"
+  echo "                           MODE==run (run a script) [default: shell]"
+  echo "  -r | --repository CFG  - configuration file or identifier defining the"
+  echo "                           repository to use [default: EESSI-pilot via"
   echo "                          container configuration]"
-  echo "  -u | --resume DIR/TGZ - resume a previous run from a directory or tarball,"
-  echo "                          where DIR points to a previously used tmp directory"
-  echo "                          (check for output 'Using DIR as tmp ...' of a previous"
-  echo "                          run) and TGZ is the path to a tarball which is"
-  echo "                          unpacked the tmp dir stored on the local storage space"
-  echo "                          (see option --storage above) [default: not set]"
-  echo "  -s | --save DIR/TGZ   - save contents of tmp directory to a tarball in"
-  echo "                          directory DIR or provided with the fixed full path TGZ"
-  echo "                          when a directory is provided, the format of the"
-  echo "                          tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz"
-  echo "                          [default: not set]"
-  echo "  -v | --verbose        - display more information [default: false]"
+  echo "  -u | --resume DIR/TGZ  - resume a previous run from a directory or tarball,"
+  echo "                           where DIR points to a previously used tmp directory"
+  echo "                           (check for output 'Using DIR as tmp ...' of a previous"
+  echo "                           run) and TGZ is the path to a tarball which is"
+  echo "                           unpacked the tmp dir stored on the local storage space"
+  echo "                           (see option --storage above) [default: not set]"
+  echo "  -s | --save DIR/TGZ    - save contents of tmp directory to a tarball in"
+  echo "                           directory DIR or provided with the fixed full path TGZ"
+  echo "                           when a directory is provided, the format of the"
+  echo "                           tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz"
+  echo "                           [default: not set]"
+  echo "  -v | --verbose         - display more information [default: false]"
+  echo "  -x | --http-proxy URL  - provides URL for the env variable http_proxy"
+  echo "                           [default: not set]; uses env var \$http_proxy if set"
+  echo "  -y | --https-proxy URL - provides URL for the env variable https_proxy"
+  echo "                           [default: not set]; uses env var \$https_proxy if set"
   echo
   echo " If value for --mode is 'run', the SCRIPT provided is executed."
-  #echo
-  #echo " FEATURES/OPTIONS to be implemented:"
-  #echo "  -d | --dry-run           -  run script except for executing the container,"
-  #echo "                              print information about setup [default: false]"
-  #echo "  -x | --http-proxy URL    -  provides URL for the env variable http_proxy"
-  #echo "                              [default: not set]"
-  #echo "  -y | --https-proxy URL   -  provides URL for the env variable https_proxy"
-  #echo "                              [default: not set]"
+  echo
+  echo " FEATURES/OPTIONS to be implemented:"
+  echo "  -d | --dry-run         -  run script except for executing the container,"
+  echo "                            print information about setup [default: false]"
 }
 
 # set defaults for command line arguments
@@ -109,8 +109,8 @@ MODE="shell"
 REPOSITORY="EESSI-pilot"
 RESUME=
 SAVE=
-#HTTP_PROXY=
-#HTTPS_PROXY=
+HTTP_PROXY=${http_proxy:-}
+HTTPS_PROXY=${https_proxy:-}
 
 POSITIONAL_ARGS=()
 
@@ -156,16 +156,16 @@ while [[ $# -gt 0 ]]; do
       VERBOSE=1
       shift 1
       ;;
-#    -x|--http-proxy)
-#      HTTP_PROXY="$2"
-#      export http_proxy=${HTTP_PROXY}
-#      shift 2
-#      ;;
-#    -y|--https-proxy)
-#      HTTPS_PROXY="$2"
-#      export https_proxy=${HTTPS_PROXY}
-#      shift 2
-#      ;;
+    -x|--http-proxy)
+      HTTP_PROXY="$2"
+      export http_proxy=${HTTP_PROXY}
+      shift 2
+      ;;
+    -y|--https-proxy)
+      HTTPS_PROXY="$2"
+      export https_proxy=${HTTPS_PROXY}
+      shift 2
+      ;;
     -*|--*)
       fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}"
       ;;
@@ -311,9 +311,9 @@ BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/ru
 BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp"
 [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}"
 
-# set up repository config (always create cfg dir and populate it with info when
+# set up repository config (always create directory repos_cfg and populate it with info when
 # arg -r|--repository is used)
-mkdir -p ${EESSI_TMPDIR}/cfg
+mkdir -p ${EESSI_TMPDIR}/repos_cfg
 if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then
   # need to source defaults as late as possible (see other sourcing below)
   source ${TOPDIR}/init/eessi_defaults
@@ -370,13 +370,13 @@ else
 
   # only unpack config_bundle if we're not resuming from a previous run
   if [[ -z ${RESUME} ]]; then
-    tar xf ${config_bundle} -C ${EESSI_TMPDIR}/cfg
+    tar xf ${config_bundle} -C ${EESSI_TMPDIR}/repos_cfg
   fi
 
   for src in "${!cfg_file_map[@]}"
   do
     target=${cfg_file_map[${src}]}
-    BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/cfg/${src}:${target}"
+    BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}"
   done
   export EESSI_PILOT_VERSION_OVERRIDE=${repo_version}
   export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}"
@@ -432,6 +432,13 @@ else
 fi
 [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}"
 
+# pass $EESSI_SOFTWARE_SUBDIR_OVERRIDE into build container (if set)
+if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
+    export SINGULARITYENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
+    # also specify via $APPTAINERENV_* (future proof, cfr. https://apptainer.org/docs/user/latest/singularity_compatibility.html#singularity-environment-variable-compatibility)
+    export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
+fi
+
 echo "Launching container with command (next line):"
 echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
 # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level

From 837234107d28cd9b2d759a66af8f7e2ec5244923 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 18 Feb 2023 20:55:30 +0100
Subject: [PATCH 074/157] provide CVMFS_HTTP_PROXY if necessary

cherry-picked via commit b5c07eeb99e77191d818e64788b0d419031cf526
NOTE, only applied changes to eessi_container.sh
---
 eessi_container.sh | 39 ++++++++++++++++++++++++++++-----------
 1 file changed, 28 insertions(+), 11 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 3045fd5a89..9238d3c793 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -24,8 +24,8 @@
 # 2. set up host storage/tmp
 # 3. set up common vars and directories
 # 4. set up vars specific to a scenario
-# 5. initialize host storage/tmp from previous run if provided
-# 6. run container
+# 5. run container
+# 6. save tmp (if requested)
 
 # -. initial settings & exit codes
 TOPDIR=$(dirname $(realpath $0))
@@ -279,7 +279,7 @@ fi
 #      |-overlay-upper
 #      |-overlay-work
 #      |-home
-#      |-cfg
+#      |-repos_cfg
 
 # tmp dir for EESSI
 EESSI_TMPDIR=${EESSI_HOST_STORAGE}
@@ -361,9 +361,9 @@ else
   cfg_init_file_map "${config_map}"
   [[ ${VERBOSE} -eq 1 ]] && cfg_print_map
 
-  # TODO use information to set up dir ${EESSI_TMPDIR}/cfg,
-  #      define BIND mounts and override repo name and version
-  # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/cfg
+  # use information to set up dir ${EESSI_TMPDIR}/repos_cfg,
+  #     define BIND mounts and override repo name and version
+  # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/repos_cfg
   if [[ ! -r ${config_bundle} ]]; then
     fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE}
   fi
@@ -384,6 +384,26 @@ else
   source ${TOPDIR}/init/eessi_defaults
 fi
 
+# if http_proxy is not empty, we assume that the machine accesses internet
+# via a proxy. then we need to add CVMFS_HTTP_PROXY to
+# ${EESSI_TMPDIR}/repos_cfg/default.local on host (and possibly add a BIND
+# MOUNT if it was not yet in BIND_PATHS)
+if [[ ! -z ${http_proxy} ]]; then
+    # TODO tolerate other formats for proxy URLs, for now assume format is
+    # http://SOME_HOSTNAME:SOME_PORT
+    PROXY_HOST_AND_PORT=${http_proxy#http:\/\//} # strip http://
+    PROXY_PORT=${PROXY_HOST_AND_PORT#.*:/} # remove hostname: to get port
+    HTTP_PROXY_HOSTNAME=${PROXY_HOST_AND_PORT%:${PROX_PORT}/}
+    HTTP_PROXY_IPV4=$(get_ipv4_address ${HTTP_PROXY_HOSTNAME}) 
+    echo "CVMFS_HTTP_PROXY=\"${http_proxy}|" \
+         "http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \
+       >> ${EESSI_TMPDIR}/repos_cfg/default.local
+    cat ${EESSI_TMPDIR}/repos_cfg/default.local
+    # if default.local is not BIND mounted into container, add it to BIND_PATHS
+    if [[ ${BIND_PATHS} !~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then
+        export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local"
+    fi
+fi
 
 # 4. set up vars and dirs specific to a scenario
 
@@ -420,10 +440,7 @@ if [[ "${ACCESS}" == "rw" ]]; then
 fi
 
 
-# 5. initialize host storage/tmp from previous run if provided
-
-
-# 6. run container
+# 5. run container
 # final settings
 if [[ -z ${SINGULARITY_BIND} ]]; then
     export SINGULARITY_BIND="${BIND_PATHS}"
@@ -446,7 +463,7 @@ echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
 singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"
 exit_code=$?
 
-# 7. save tmp if requested (arg -s|--save)
+# 6. save tmp if requested (arg -s|--save)
 if [[ ! -z ${SAVE} ]]; then
   # Note, for now we don't try to be smart and record in any way the OS and
   #   ARCH which might have been used internally, eg, when software packages

From 3f1aed9a2b5da7fa7e017e415d24a412dbfa18fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= <thomas.roblitz@uib.no>
Date: Fri, 27 Jan 2023 22:41:36 +0100
Subject: [PATCH 075/157] add functions to derive hostname and port number from
 proxy URL

---
 scripts/utils.sh | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/scripts/utils.sh b/scripts/utils.sh
index db533adc12..fb4eff1e3f 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -64,6 +64,30 @@ function get_path_for_tool {
     fi
 }
 
+function get_host_from_url {
+    url=$1
+    re="(http|https)://([^/:]+)"
+    if [[ $url =~ $re ]]; then
+        echo ${BASH_REMATCH[2]}
+        return 0
+    else
+        echo ""
+        return 1
+    fi 
+}
+
+function get_port_from_url {
+    url=$1
+    re="(http|https)://[^:]+:([0-9]+)"
+    if [[ $url =~ $re ]]; then
+        echo ${BASH_REMATCH[2]}
+        return 0
+    else
+        echo ""
+        return 1
+    fi 
+}
+
 function get_ipv4_address {
     hname=$1
     hipv4=$(grep ${hname} /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1)

From b7abfc6e7de85e2b46e3241d8946a1211dcde568 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= <thomas.roblitz@uib.no>
Date: Fri, 27 Jan 2023 22:43:27 +0100
Subject: [PATCH 076/157] add function to determine config sections

---
 cfg_files.sh | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/cfg_files.sh b/cfg_files.sh
index 885ebd0877..ffd29db50a 100644
--- a/cfg_files.sh
+++ b/cfg_files.sh
@@ -47,9 +47,7 @@ function cfg_load {
     else
       val=$(cfg_get_key_value $line)
       # trim leading and trailing spaces as well
-      #cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//')
       cur_key=$(echo $val | cut -f1 -d'=' | cfg_trim_spaces)
-      #cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//')
       cur_val=$(echo $val | cut -f2 -d'=' | cfg_trim_spaces)
       if [[ -n "$cur_key" ]]; then
         # section + key is the associative in bash array, the field separator is space
@@ -69,6 +67,20 @@ function cfg_print {
   done
 }
 
+function cfg_sections {
+  declare -A sections
+  for key in "${!cfg_repos[@]}"
+  do
+    # extract section from the associative key
+    section=$(echo $key | cut -f1 -d ' ')
+    sections[${section}]=1
+  done
+  for repo in "${!sections[@]}"
+  do
+      echo "${repo}"
+  done
+}
+
 function cfg_get_value {
   section=$1
   key=$2

From 9b7b8a4db00261b0fe2bd40152d7adacde11ffe3 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 18 Feb 2023 21:17:11 +0100
Subject: [PATCH 077/157] add parameter to list repositories, fix bugs

fixed bugs

- determining IP address for proxy host was wrong
- handling of positional parameters for executing commands with arguments was wrong
---
 eessi_container.sh | 36 ++++++++++++++++++++++++++++--------
 1 file changed, 28 insertions(+), 8 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 9238d3c793..06b77a5acf 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -70,6 +70,7 @@ display_help() {
   echo "  -h | --help            - display this usage information [default: false]"
   echo "  -g | --storage DIR     - directory space on host machine (used for"
   echo "                           temporary data) [default: 1. TMPDIR, 2. /tmp]"
+  echo "  -l | --list-repos      - list available repository identifiers [default: false]"
   echo "  -m | --mode MODE       - with MODE==shell (launch interactive shell) or"
   echo "                           MODE==run (run a script) [default: shell]"
   echo "  -r | --repository CFG  - configuration file or identifier defining the"
@@ -105,6 +106,7 @@ CONTAINER="docker://ghcr.io/eessi/build-node:debian11"
 #DRY_RUN=0
 VERBOSE=0
 STORAGE=
+LIST_REPOS=0
 MODE="shell"
 REPOSITORY="EESSI-pilot"
 RESUME=
@@ -136,6 +138,10 @@ while [[ $# -gt 0 ]]; do
       display_help
       exit 0
       ;;
+    -l|--list-repos)
+      LIST_REPOS=1
+      shift 1
+      ;;
     -m|--mode)
       MODE="$2"
       shift 2
@@ -178,6 +184,17 @@ done
 
 set -- "${POSITIONAL_ARGS[@]}"
 
+if [[ ${LIST_REPOS} -eq 1 ]]; then
+    echo "Repositories defined in the config file '${EESSI_REPOS_CFG_FILE}':"
+    echo "    EESSI-pilot [default]"
+    cfg_load ${EESSI_REPOS_CFG_FILE}
+    sections=$(cfg_sections)
+    while IFS= read -r repo_id
+    do
+        echo "    ${repo_id}"
+    done <<< "${sections}"
+    exit 0
+fi
 
 # 1. check if argument values are valid
 # (arg -a|--access) check if ACCESS is supported
@@ -390,17 +407,20 @@ fi
 # MOUNT if it was not yet in BIND_PATHS)
 if [[ ! -z ${http_proxy} ]]; then
     # TODO tolerate other formats for proxy URLs, for now assume format is
-    # http://SOME_HOSTNAME:SOME_PORT
-    PROXY_HOST_AND_PORT=${http_proxy#http:\/\//} # strip http://
-    PROXY_PORT=${PROXY_HOST_AND_PORT#.*:/} # remove hostname: to get port
-    HTTP_PROXY_HOSTNAME=${PROXY_HOST_AND_PORT%:${PROX_PORT}/}
-    HTTP_PROXY_IPV4=$(get_ipv4_address ${HTTP_PROXY_HOSTNAME}) 
-    echo "CVMFS_HTTP_PROXY=\"${http_proxy}|" \
-         "http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \
+    # http://SOME_HOSTNAME:SOME_PORT/
+    [[ ${VERBOSE} -eq 1 ]] && echo "http_proxy='${http_proxy}'"
+    PROXY_HOST=$(get_host_from_url ${http_proxy})
+    [[ ${VERBOSE} -eq 1 ]] && echo "PROXY_HOST='${PROXY_HOST}'"
+    PROXY_PORT=$(get_port_from_url ${http_proxy})
+    [[ ${VERBOSE} -eq 1 ]] && echo "PROXY_PORT='${PROXY_PORT}'"
+    HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST}) 
+    [[ ${VERBOSE} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'"
+    echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \
        >> ${EESSI_TMPDIR}/repos_cfg/default.local
     cat ${EESSI_TMPDIR}/repos_cfg/default.local
+
     # if default.local is not BIND mounted into container, add it to BIND_PATHS
-    if [[ ${BIND_PATHS} !~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then
+    if [[ ! ${BIND_PATHS} =~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then
         export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local"
     fi
 fi

From 4a414f593d6ff02629d1374e67006dcd1a85f71c Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 18 Feb 2023 21:26:19 +0100
Subject: [PATCH 078/157] replace FILE with DIR in
 EESSI_REPOS_CFG_FILE_OVERRIDE

NOTE only applied changes to eessi_container.sh
---
 eessi_container.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 06b77a5acf..d59bb38687 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -52,8 +52,8 @@ CVMFS_VAR_LIB="var-lib-cvmfs"
 CVMFS_VAR_RUN="var-run-cvmfs"
 
 # repository cfg file, default name (default location: $PWD)
-#   can be overwritten by setting env var EESSI_REPOS_CFG_FILE_OVERRIDE
-export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}"
+#   can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE
+export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=.}/repos.cfg"
 
 
 # 0. parse args

From 37abf4f8aab9d138ad763cd457d2476bef16a6aa Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 29 Jan 2023 19:56:41 +0100
Subject: [PATCH 079/157] fix whitespace issues

---
 cfg_files.sh        | 1 -
 eessi_container.sh  | 2 +-
 init/eessi_defaults | 1 -
 scripts/utils.sh    | 4 ++--
 4 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/cfg_files.sh b/cfg_files.sh
index ffd29db50a..57ea2f7c03 100644
--- a/cfg_files.sh
+++ b/cfg_files.sh
@@ -165,4 +165,3 @@ function cfg_print_map {
     echo "${index} --> ${cfg_file_map[${index}]}"
   done
 }
-
diff --git a/eessi_container.sh b/eessi_container.sh
index d59bb38687..1a2e61227b 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -413,7 +413,7 @@ if [[ ! -z ${http_proxy} ]]; then
     [[ ${VERBOSE} -eq 1 ]] && echo "PROXY_HOST='${PROXY_HOST}'"
     PROXY_PORT=$(get_port_from_url ${http_proxy})
     [[ ${VERBOSE} -eq 1 ]] && echo "PROXY_PORT='${PROXY_PORT}'"
-    HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST}) 
+    HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST})
     [[ ${VERBOSE} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'"
     echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \
        >> ${EESSI_TMPDIR}/repos_cfg/default.local
diff --git a/init/eessi_defaults b/init/eessi_defaults
index 1b5ce07fb1..f482cbc269 100644
--- a/init/eessi_defaults
+++ b/init/eessi_defaults
@@ -10,4 +10,3 @@
 
 export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/pilot.eessi-hpc.org}"
 export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2021.12}"
-
diff --git a/scripts/utils.sh b/scripts/utils.sh
index fb4eff1e3f..d0da95e87f 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -73,7 +73,7 @@ function get_host_from_url {
     else
         echo ""
         return 1
-    fi 
+    fi
 }
 
 function get_port_from_url {
@@ -85,7 +85,7 @@ function get_port_from_url {
     else
         echo ""
         return 1
-    fi 
+    fi
 }
 
 function get_ipv4_address {

From d468a1ababa8072c4069917f4c16529cd362011f Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 18 Feb 2023 21:42:34 +0100
Subject: [PATCH 080/157] include changes made to test PR for building with the
 bot

cherry-picked via commit fce504fb466be005f1d417e96e47f442c8f29c52
NOTE, only changed eessi_container.sh
---
 eessi_container.sh | 50 ++++++++++++++++++++++++++++++++--------------
 1 file changed, 35 insertions(+), 15 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 1a2e61227b..14fb716abb 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -51,9 +51,15 @@ RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11))
 CVMFS_VAR_LIB="var-lib-cvmfs"
 CVMFS_VAR_RUN="var-run-cvmfs"
 
+# directory for tmp used inside container
+export TMP_IN_CONTAINER=/tmp
+
 # repository cfg file, default name (default location: $PWD)
 #   can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE
-export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=.}/repos.cfg"
+export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}/repos.cfg"
+# other repository cfg files in directory, default location: $PWD
+#   can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE
+export EESSI_REPOS_CFG_DIR="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}"
 
 
 # 0. parse args
@@ -199,7 +205,7 @@ fi
 # 1. check if argument values are valid
 # (arg -a|--access) check if ACCESS is supported
 if [[ "${ACCESS}" != "ro" && "${ACCESS}" != "rw" ]]; then
-  fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}"
+    fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}"
 fi
 
 # TODO (arg -c|--container) check container (is it a file or URL & access those)
@@ -211,11 +217,14 @@ fi
 
 # (arg -m|--mode) check if MODE is known
 if [[ "${MODE}" != "shell" && "${MODE}" != "run" ]]; then
-  fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}"
+    fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}"
 fi
 
 # TODO (arg -r|--repository) check if repository is known
 # REPOSITORY_ERROR_EXITCODE
+if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI-pilot" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then
+    fatal_error "arg '--repository ${REPOSITORY}' requires a cfg file at '${EESSI_REPOS_CFG_FILE}'" "${REPOSITORY_ERROR_EXITCODE}"
+fi
 
 # TODO (arg -u|--resume) check if it exists, if user has read permission,
 #      if it contains data from a previous run
@@ -249,6 +258,7 @@ if [[ ! -z ${RESUME} && -d ${RESUME} ]]; then
   #   skip creating a new tmp directory, just set environment variables
   echo "Resuming from previous run using temporary storage at ${RESUME}"
   EESSI_HOST_STORAGE=${RESUME}
+  echo "RESUME_FROM_DIR ${EESSI_HOST_STORAGE}"
 else
   # we need a tmp location (and possibly init it with ${RESUME} if it was not
   #   a directory
@@ -325,7 +335,9 @@ fi
 
 # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined)
 BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs"
-BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp"
+# provide a '/tmp' inside the container
+BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}"
+
 [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}"
 
 # set up repository config (always create directory repos_cfg and populate it with info when
@@ -381,13 +393,22 @@ else
   # use information to set up dir ${EESSI_TMPDIR}/repos_cfg,
   #     define BIND mounts and override repo name and version
   # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/repos_cfg
-  if [[ ! -r ${config_bundle} ]]; then
-    fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE}
+  # if config_bundle is relative path (no '/' at start) prepend it with
+  # EESSI_REPOS_CFG_DIR
+  config_bundle_path=
+  if [[ ! "${config_bundle}" =~ ^/ ]]; then
+      config_bundle_path=${EESSI_REPOS_CFG_DIR}/${config_bundle}
+  else
+      config_bundle_path=${config_bundle}
+  fi
+
+  if [[ ! -r ${config_bundle_path} ]]; then
+    fatal_error "config bundle '${config_bundle_path}' is not readable" ${REPOSITORY_ERROR_EXITCODE}
   fi
 
   # only unpack config_bundle if we're not resuming from a previous run
   if [[ -z ${RESUME} ]]; then
-    tar xf ${config_bundle} -C ${EESSI_TMPDIR}/repos_cfg
+    tar xf ${config_bundle_path} -C ${EESSI_TMPDIR}/repos_cfg
   fi
 
   for src in "${!cfg_file_map[@]}"
@@ -436,12 +457,8 @@ if [[ "${ACCESS}" == "ro" ]]; then
 fi
 
 if [[ "${ACCESS}" == "rw" ]]; then
-  EESSI_CVMFS_OVERLAY_UPPER=/tmp/overlay-upper
-  EESSI_CVMFS_OVERLAY_WORK=/tmp/overlay-work
   mkdir -p ${EESSI_TMPDIR}/overlay-upper
   mkdir -p ${EESSI_TMPDIR}/overlay-work
-  [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}"
-  [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}"
 
   # set environment variables for fuse mounts in Singularity container
   export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}"
@@ -450,8 +467,8 @@ if [[ "${ACCESS}" == "rw" ]]; then
 
   EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs"
   EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}"
-  EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=/tmp/overlay-upper"
-  EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=/tmp/overlay-work"
+  EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper"
+  EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work"
   EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}"
   export EESSI_PILOT_WRITABLE_OVERLAY
 
@@ -476,11 +493,13 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
 fi
 
+# if INFO is set to 1 (arg --info), add argument '-q'
+RUN_QUIET=${INFO:--q}
 echo "Launching container with command (next line):"
-echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
+echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
 # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level
 #      provided to the script
-singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"
+singularity ${RUN_QUIET} ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"
 exit_code=$?
 
 # 6. save tmp if requested (arg -s|--save)
@@ -500,6 +519,7 @@ if [[ ! -z ${SAVE} ]]; then
   fi
   tar cf ${TGZ} -C ${EESSI_TMPDIR} .
   echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')"
+  echo "RESUME_FROM_TGZ ${TGZ}"
 fi
 
 # TODO clean up tmp by default? only retain if another option provided (--retain-tmp)

From af658ee9229f36eac972bef87a36bbb6b883ef48 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 3 Feb 2023 20:58:01 +0100
Subject: [PATCH 081/157] fix issues in eessi_container.sh

---
 eessi_container.sh | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 14fb716abb..23118d0d72 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -258,7 +258,6 @@ if [[ ! -z ${RESUME} && -d ${RESUME} ]]; then
   #   skip creating a new tmp directory, just set environment variables
   echo "Resuming from previous run using temporary storage at ${RESUME}"
   EESSI_HOST_STORAGE=${RESUME}
-  echo "RESUME_FROM_DIR ${EESSI_HOST_STORAGE}"
 else
   # we need a tmp location (and possibly init it with ${RESUME} if it was not
   #   a directory
@@ -290,6 +289,7 @@ else
   EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX)
   echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)."
 fi
+echo "RESUME_FROM_DIR ${EESSI_HOST_STORAGE}"
 
 # if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE}
 if [[ ! -z ${RESUME} && -f ${RESUME} ]]; then
@@ -494,7 +494,12 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
 fi
 
 # if INFO is set to 1 (arg --info), add argument '-q'
-RUN_QUIET=${INFO:--q}
+if [[ -z ${INFO} ]]; then
+    RUN_QUIET='-q'
+else
+    RUN_QUIET=''
+fi
+
 echo "Launching container with command (next line):"
 echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
 # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level

From eb66e1c4361f458114d2186a846fa9fe56b1df16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= <thomas.roblitz@uib.no>
Date: Sat, 4 Feb 2023 15:07:23 +0100
Subject: [PATCH 082/157] fix RUN_QUIET setting

---
 eessi_container.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 23118d0d72..89d4bd98f6 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -493,8 +493,8 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
 fi
 
-# if INFO is set to 1 (arg --info), add argument '-q'
-if [[ -z ${INFO} ]]; then
+# if INFO is set to 0 (no arg --info), add argument '-q'
+if [[ ${INFO} -eq 0 ]]; then
     RUN_QUIET='-q'
 else
     RUN_QUIET=''

From 1cb4674978bee259253330d769ca220ad795202a Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 18 Feb 2023 21:59:39 +0100
Subject: [PATCH 083/157] improvements to resuming job environment

cherry-picked via commit 23e773ce0d8c6504e1440629a4178318265b73d2
NOTE only applied changes to eessi_container.sh
---
 eessi_container.sh | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 89d4bd98f6..e714f8e2d0 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -360,6 +360,9 @@ else
   #    standard EESSI repositories)
   cfg_load ${EESSI_REPOS_CFG_FILE}
 
+  # copy repos.cfg to job directory --> makes it easier to inspect the job
+  cp ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/.
+
   # cfg file should include: repo_name, repo_version, config_bundle,
   #   map { local_filepath -> container_filepath }
   #
@@ -493,8 +496,8 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
 fi
 
-# if INFO is set to 0 (no arg --info), add argument '-q'
-if [[ ${INFO} -eq 0 ]]; then
+# if INFO is set to 1 (arg --info), add argument '-q'
+if [[ -z ${INFO} ]]; then
     RUN_QUIET='-q'
 else
     RUN_QUIET=''

From 0c465da3de12ed1bb7494651031d64d7f17928e4 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Tue, 7 Feb 2023 19:27:07 +0100
Subject: [PATCH 084/157] address requested change

---
 eessi_container.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index e714f8e2d0..e8698cd719 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -496,8 +496,8 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
 fi
 
-# if INFO is set to 1 (arg --info), add argument '-q'
-if [[ -z ${INFO} ]]; then
+# if INFO is set to 0 (no arg --info), add argument '-q'
+if [[ ${INFO} -eq 0 ]]; then
     RUN_QUIET='-q'
 else
     RUN_QUIET=''

From 26ab98c1f1cc901d6905424c9fdb2f5c9395ed79 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 18 Feb 2023 22:16:36 +0100
Subject: [PATCH 085/157] cleaning up leftovers after adding updates from NESSI

---
 eessi_container.sh | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index e8698cd719..e7ad625a15 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -100,10 +100,6 @@ display_help() {
   echo "                           [default: not set]; uses env var \$https_proxy if set"
   echo
   echo " If value for --mode is 'run', the SCRIPT provided is executed."
-  echo
-  echo " FEATURES/OPTIONS to be implemented:"
-  echo "  -d | --dry-run         -  run script except for executing the container,"
-  echo "                            print information about setup [default: false]"
 }
 
 # set defaults for command line arguments
@@ -496,8 +492,8 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
 fi
 
-# if INFO is set to 0 (no arg --info), add argument '-q'
-if [[ ${INFO} -eq 0 ]]; then
+# if VERBOSE is set to 0 (no arg --verbose), add argument '-q'
+if [[ ${VERBOSE} -eq 0 ]]; then
     RUN_QUIET='-q'
 else
     RUN_QUIET=''
@@ -505,8 +501,6 @@ fi
 
 echo "Launching container with command (next line):"
 echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
-# TODO for now we run singularity with '-q' (quiet), later adjust this to the log level
-#      provided to the script
 singularity ${RUN_QUIET} ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"
 exit_code=$?
 

From a44088e4c33694c0047a6e5874df780e3c2ff15e Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 18 Feb 2023 22:21:42 +0100
Subject: [PATCH 086/157] moving cfg_files.sh to scripts dir

---
 eessi_container.sh                   | 2 +-
 cfg_files.sh => scripts/cfg_files.sh | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename cfg_files.sh => scripts/cfg_files.sh (100%)

diff --git a/eessi_container.sh b/eessi_container.sh
index e7ad625a15..6d3fbc0adc 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -31,7 +31,7 @@
 TOPDIR=$(dirname $(realpath $0))
 
 source ${TOPDIR}/scripts/utils.sh
-source ${TOPDIR}/cfg_files.sh
+source ${TOPDIR}/scripts/cfg_files.sh
 
 # exit codes: bitwise shift codes to allow for combination of exit codes
 # ANY_ERROR_EXITCODE is sourced from ${TOPDIR}/scripts/utils.sh
diff --git a/cfg_files.sh b/scripts/cfg_files.sh
similarity index 100%
rename from cfg_files.sh
rename to scripts/cfg_files.sh

From bbbad0dce469299e4168a54a939facc4c1be7c60 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 20 Feb 2023 10:07:03 +0100
Subject: [PATCH 087/157] improved handling of container cache + explicit pull
 of image

cherry-picked via commit bfb1b29103dc309a12573f3ec9c247d432f08f73
Note, only part for eessi_container.sh included here
---
 eessi_container.sh | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 6d3fbc0adc..cdd2c017fb 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -310,10 +310,23 @@ mkdir -p ${EESSI_TMPDIR}
 [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}"
 
 # configure Singularity
-export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache
-mkdir -p ${SINGULARITY_CACHEDIR}
+if [[ -z ${SINGULARITY_CACHEDIR} ]]; then
+    export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache
+    mkdir -p ${SINGULARITY_CACHEDIR}
+fi
 [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}"
 
+# pull & convert image and reset CONTAINER
+CONTAINER_URL_FMT=".*://(.*)"
+if [[ ${CONTAINER} == ${CONTAINER_URL_FMT} ]]; then
+    CONTAINER_IMG=${BASH_REMATCH[1]//[:-]/_}.sif
+    singularity pull ${CONTAINER_IMG} ${CONTAINER}
+    if [[ -x ${CONTAINER_IMG} ]]; then
+        CONTAINER="${PWD}/${CONTAINER_IMG}"
+    fi
+fi
+[[ ${INFO} -eq 1 ]] && echo "CONTAINER='${CONTAINER}'"
+
 # set env vars and create directories for CernVM-FS
 EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB}
 EESSI_CVMFS_VAR_RUN=${EESSI_TMPDIR}/${CVMFS_VAR_RUN}
@@ -326,8 +339,8 @@ mkdir -p ${EESSI_CVMFS_VAR_RUN}
 if [[ -z ${SINGULARITY_HOME} ]]; then
   export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}"
   mkdir -p ${EESSI_TMPDIR}/home
-  [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}"
 fi
+[[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}"
 
 # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined)
 BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs"

From 15ae58df240b1b1396611ee6fb5b5dbd50d4e793 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 20 Feb 2023 10:30:17 +0100
Subject: [PATCH 088/157] use VERBOSE instead of INFO

---
 .../workflows/test_eessi_container_script.yml | 22 +++++++++++++++++++
 eessi_container.sh                            |  2 +-
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index 7678aedc64..5a44bae155 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -11,6 +11,8 @@ jobs:
       matrix:
         SCRIPT_TEST:
         - help
+        - listrepos_default
+        - listrepos_custom
         - run
         - shell
         - container
@@ -39,6 +41,26 @@ jobs:
               if [[ ${{matrix.SCRIPT_TEST}} == 'help' ]]; then
                 ./eessi_container.sh --help
 
+              # test use of --list-repos without custom repos.cfg
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_default' ]]; then
+                outfile=out_listrepos.txt
+                ./eessi_container.sh --verbose --list-repos | tee ${outfile}
+                grep "EESSI-pilot" ${outfile}
+
+              # test use of --list-repos with custom repos.cfg
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_custom' ]]; then
+                outfile=out_listrepos.txt
+                outfile2=out_listrepos_2.txt
+                mkdir -p ${PWD}/cfg
+                echo "[EESSI/2021.12]" > cfg/repos.cfg
+                echo "[EESSI/2023.02]" >> cfg/repos.cfg
+                ./eessi_container.sh --verbose --list-repos | tee ${outfile}
+                grep "EESSI-pilot" ${outfile}
+
+                export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
+                ./eessi_container.sh --verbose --list-repos | tee ${outfile2}
+                grep "[EESSI/2023.02]" ${outfile2}
+
               # test use of --mode run
               elif [[ ${{matrix.SCRIPT_TEST}} == 'run' ]]; then
                 outfile=out_run.txt
diff --git a/eessi_container.sh b/eessi_container.sh
index cdd2c017fb..76a4d34143 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -325,7 +325,7 @@ if [[ ${CONTAINER} == ${CONTAINER_URL_FMT} ]]; then
         CONTAINER="${PWD}/${CONTAINER_IMG}"
     fi
 fi
-[[ ${INFO} -eq 1 ]] && echo "CONTAINER='${CONTAINER}'"
+[[ ${VERBOSE} -eq 1 ]] && echo "CONTAINER=${CONTAINER}"
 
 # set env vars and create directories for CernVM-FS
 EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB}

From 764e7138b0f8dfd82c81c1647bf78d04da1aec68 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 20 Feb 2023 10:48:38 +0100
Subject: [PATCH 089/157] fix test for --list-repos; improve output for
 --list-repos

---
 .../workflows/test_eessi_container_script.yml   |  2 ++
 eessi_container.sh                              | 17 ++++++++++-------
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index 5a44bae155..74cfd1fb10 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -53,7 +53,9 @@ jobs:
                 outfile2=out_listrepos_2.txt
                 mkdir -p ${PWD}/cfg
                 echo "[EESSI/2021.12]" > cfg/repos.cfg
+                echo "repo_version = 2021.12" >> cfg/repos.cfg
                 echo "[EESSI/2023.02]" >> cfg/repos.cfg
+                echo "repo_version = 2023.02" >> cfg/repos.cfg
                 ./eessi_container.sh --verbose --list-repos | tee ${outfile}
                 grep "EESSI-pilot" ${outfile}
 
diff --git a/eessi_container.sh b/eessi_container.sh
index 76a4d34143..dec082ce04 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -186,15 +186,18 @@ done
 
 set -- "${POSITIONAL_ARGS[@]}"
 
+
 if [[ ${LIST_REPOS} -eq 1 ]]; then
-    echo "Repositories defined in the config file '${EESSI_REPOS_CFG_FILE}':"
+    echo "Listing available repositories with format 'name [source]':"
     echo "    EESSI-pilot [default]"
-    cfg_load ${EESSI_REPOS_CFG_FILE}
-    sections=$(cfg_sections)
-    while IFS= read -r repo_id
-    do
-        echo "    ${repo_id}"
-    done <<< "${sections}"
+    if [[ -r ${EESSI_REPOS_CFG_FILE} ]]; then
+        cfg_load ${EESSI_REPOS_CFG_FILE}
+        sections=$(cfg_sections)
+        while IFS= read -r repo_id
+        do
+            echo "    ${repo_id} [${EESSI_REPOS_CFG_FILE}]"
+        done <<< "${sections}"
+    fi
     exit 0
 fi
 

From 47cc1c6be44896675949567d88f5ae85a998b64f Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Tue, 21 Feb 2023 21:27:56 +0100
Subject: [PATCH 090/157] implemented requested changes

---
 .../workflows/test_eessi_container_script.yml |  8 +--
 eessi_container.sh                            | 59 ++++++++++++++-----
 2 files changed, 48 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index 74cfd1fb10..d4a9dfd53e 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -52,10 +52,10 @@ jobs:
                 outfile=out_listrepos.txt
                 outfile2=out_listrepos_2.txt
                 mkdir -p ${PWD}/cfg
-                echo "[EESSI/2021.12]" > cfg/repos.cfg
-                echo "repo_version = 2021.12" >> cfg/repos.cfg
-                echo "[EESSI/2023.02]" >> cfg/repos.cfg
-                echo "repo_version = 2023.02" >> cfg/repos.cfg
+                echo "[EESSI/20AB.CD]" > cfg/repos.cfg
+                echo "repo_version = 20AB.CD" >> cfg/repos.cfg
+                echo "[EESSI/20HT.TP]" >> cfg/repos.cfg
+                echo "repo_version = 20HT.TP" >> cfg/repos.cfg
                 ./eessi_container.sh --verbose --list-repos | tee ${outfile}
                 grep "EESSI-pilot" ${outfile}
 
diff --git a/eessi_container.sh b/eessi_container.sh
index dec082ce04..86a510d56e 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -54,12 +54,11 @@ CVMFS_VAR_RUN="var-run-cvmfs"
 # directory for tmp used inside container
 export TMP_IN_CONTAINER=/tmp
 
-# repository cfg file, default name (default location: $PWD)
-#   can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE
-export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}/repos.cfg"
-# other repository cfg files in directory, default location: $PWD
-#   can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE
+# repository cfg directory and file
+#   directory: default $PWD or EESSI_REPOS_CFG_DIR_OVERRIDE if set
+#   file: directory + '/repos.cfg'
 export EESSI_REPOS_CFG_DIR="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}"
+export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR}/repos.cfg"
 
 
 # 0. parse args
@@ -288,7 +287,6 @@ else
   EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX)
   echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)."
 fi
-echo "RESUME_FROM_DIR ${EESSI_HOST_STORAGE}"
 
 # if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE}
 if [[ ! -z ${RESUME} && -f ${RESUME} ]]; then
@@ -312,21 +310,52 @@ EESSI_TMPDIR=${EESSI_HOST_STORAGE}
 mkdir -p ${EESSI_TMPDIR}
 [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}"
 
-# configure Singularity
+# configure Singularity: if SINGULARITY_CACHEDIR is already defined, use that
+#   a global SINGULARITY_CACHEDIR would ensure that we don't consume
+#   storage space again and again for the container & also speed-up
+#   launch times across different sessions
 if [[ -z ${SINGULARITY_CACHEDIR} ]]; then
     export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache
     mkdir -p ${SINGULARITY_CACHEDIR}
 fi
 [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}"
 
-# pull & convert image and reset CONTAINER
+# we try our best to make sure that we retain access to the container image in
+# a subsequent session ("best effort" only because pulling or copying operations
+# can fail ... in those cases the script may still succeed, but it is not
+# guaranteed that we have access to the same container when resuming later on)
+# - if CONTAINER references an image in a registry, pull & convert image
+#   and store it in ${EESSI_TMPDIR}
+#   + however, only pull image if there is no matching image in ${EESSI_TMPDIR} yet
+# - if CONTAINER references an image file, copy it to ${EESSI_TMPDIR}
+#   + however, only copy it if its base name does not yet exist in ${EESSI_TMPDIR}
+# - if the image file created (pulled or copied) or resumed exists in
+#   ${EESSI_TMPDIR}, let CONTAINER point to it
+#   + thus subsequent singularity commands in this script would just use the
+#     image file in EESSI_TMPDIR or the originally given source (some URL or
+#     path to an image file)
+CONTAINER_IMG=
 CONTAINER_URL_FMT=".*://(.*)"
-if [[ ${CONTAINER} == ${CONTAINER_URL_FMT} ]]; then
+if [[ ${CONTAINER} =~ ${CONTAINER_URL_FMT} ]]; then
+    # replace : and - with _ in match (everything after ://) and append .sif
     CONTAINER_IMG=${BASH_REMATCH[1]//[:-]/_}.sif
-    singularity pull ${CONTAINER_IMG} ${CONTAINER}
-    if [[ -x ${CONTAINER_IMG} ]]; then
-        CONTAINER="${PWD}/${CONTAINER_IMG}"
+    # pull container to ${EESSI_TMPDIR} if it is not there yet (i.e. when
+    # resuming from a previous session)
+    if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then
+        singularity pull ${EESSI_TMPDIR}/${CONTAINER_IMG} ${CONTAINER}
     fi
+else
+    # determine file name as basename of CONTAINER
+    CONTAINER_IMG=$(basename ${CONTAINER})
+    # copy image file to ${EESSI_TMPDIR} if it is not there yet (i.e. when
+    # resuming from a previous session)
+    if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then
+        cp -a ${CONTAINER} ${EESSI_TMPDIR}/.
+    fi
+fi
+# let CONTAINER point to the pulled, copied or resumed image file
+if [[ -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then
+    CONTAINER="${EESSI_TMPDIR}/${CONTAINER_IMG}"
 fi
 [[ ${VERBOSE} -eq 1 ]] && echo "CONTAINER=${CONTAINER}"
 
@@ -373,7 +402,7 @@ else
   cfg_load ${EESSI_REPOS_CFG_FILE}
 
   # copy repos.cfg to job directory --> makes it easier to inspect the job
-  cp ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/.
+  cp -a ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/.
 
   # cfg file should include: repo_name, repo_version, config_bundle,
   #   map { local_filepath -> container_filepath }
@@ -453,7 +482,8 @@ if [[ ! -z ${http_proxy} ]]; then
     [[ ${VERBOSE} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'"
     echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \
        >> ${EESSI_TMPDIR}/repos_cfg/default.local
-    cat ${EESSI_TMPDIR}/repos_cfg/default.local
+    [[ ${VERBOSE} -eq 1 ]] && echo "contents of default.local"
+    [[ ${VERBOSE} -eq 1 ]] && cat ${EESSI_TMPDIR}/repos_cfg/default.local
 
     # if default.local is not BIND mounted into container, add it to BIND_PATHS
     if [[ ! ${BIND_PATHS} =~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then
@@ -537,7 +567,6 @@ if [[ ! -z ${SAVE} ]]; then
   fi
   tar cf ${TGZ} -C ${EESSI_TMPDIR} .
   echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')"
-  echo "RESUME_FROM_TGZ ${TGZ}"
 fi
 
 # TODO clean up tmp by default? only retain if another option provided (--retain-tmp)

From 712c40e567842da1c50b8c196fad0c9c5d30a69a Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 22 Feb 2023 22:54:43 +0100
Subject: [PATCH 091/157] improvements to resuming job environment

cherry-picked via 23e773ce0d8c6504e1440629a4178318265b73d2
Note only applied part for bot/build.sh
---
 bot/build.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 8565511bfa..2c15b904f3 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -141,7 +141,7 @@ REPOSITORY_OPT=
 if [[ ! -z ${REPOSITORY} ]]; then
     REPOSITORY_OPT="--repository ${REPOSITORY}"
 fi
-mkdir -p previous_tmp
+mkdir -p previous_tmp/{build_step,tarball_step}
 build_outerr=$(mktemp build.outerr.XXXX)
 echo "Executing command to build software:"
 echo "./eessi_container.sh --access rw"
@@ -151,7 +151,7 @@ echo "                     ${HTTPS_PROXY_OPT}"
 echo "                     --info"
 echo "                     --mode run"
 echo "                     ${REPOSITORY_OPT}"
-echo "                     --save ${PWD}/previous_tmp"
+echo "                     --save ${PWD}/previous_tmp/build_step"
 echo "                     --storage ${STORAGE}"
 echo "                     ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${build_outerr}"
 # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
@@ -163,7 +163,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
                      --info \
                      --mode run \
                      ${REPOSITORY_OPT} \
-                     --save ${PWD}/previous_tmp \
+                     --save ${PWD}/previous_tmp/build_step \
                      --storage ${STORAGE} \
                      ./install_software_layer.sh "$@" 2>&1 | tee -a ${build_outerr}
 
@@ -190,7 +190,7 @@ echo "                     --info"
 echo "                     --mode run"
 echo "                     ${REPOSITORY_OPT}"
 echo "                     --resume ${BUILD_TMPDIR}"
-echo "                     --save ${PWD}/previous_tmp"
+echo "                     --save ${PWD}/previous_tmp/tarball_step"
 echo "                     ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
 ./eessi_container.sh --access rw \
                      ${CONTAINER_OPT} \
@@ -200,7 +200,7 @@ echo "                     ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT
                      --mode run \
                      ${REPOSITORY_OPT} \
                      --resume ${BUILD_TMPDIR} \
-                     --save ${PWD}/previous_tmp \
+                     --save ${PWD}/previous_tmp/tarball_step \
                      ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
 
 exit 0

From 16d98572d4148049a3a9f6a4258467eaf785d9f4 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 22 Feb 2023 23:00:42 +0100
Subject: [PATCH 092/157] improved handling of container cache + explicit pull
 of image

cherry-picked via bfb1b29103dc309a12573f3ec9c247d432f08f73
Note, only applied part for bot/build.sh
---
 bot/build.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/bot/build.sh b/bot/build.sh
index 2c15b904f3..06503e59a0 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -56,6 +56,12 @@ LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE})
 echo "LOCAL_TMP='${LOCAL_TMP}'"
 # TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided
 
+SINGULARITY_CACHEDIR=$(${YQ} '.site_config.container_cachedir // ""' ${JOB_CFG_FILE})
+echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'"
+if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then
+    export SINGULARITY_CACHEDIR
+fi
+
 echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> "
 # replace any env variable in ${LOCAL_TMP} with its
 #   current value (e.g., a value that is local to the job)

From 9e8ca62cab90b013c6fb1b55180e37592aea02b5 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 22 Feb 2023 23:02:35 +0100
Subject: [PATCH 093/157] changed --info -> --verbose

---
 bot/build.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 06503e59a0..77aa2d9077 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -154,7 +154,7 @@ echo "./eessi_container.sh --access rw"
 echo "                     ${CONTAINER_OPT}"
 echo "                     ${HTTP_PROXY_OPT}"
 echo "                     ${HTTPS_PROXY_OPT}"
-echo "                     --info"
+echo "                     --verbose"
 echo "                     --mode run"
 echo "                     ${REPOSITORY_OPT}"
 echo "                     --save ${PWD}/previous_tmp/build_step"
@@ -166,7 +166,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
                      ${CONTAINER_OPT} \
                      ${HTTP_PROXY_OPT} \
                      ${HTTPS_PROXY_OPT} \
-                     --info \
+                     --verbose \
                      --mode run \
                      ${REPOSITORY_OPT} \
                      --save ${PWD}/previous_tmp/build_step \
@@ -192,7 +192,7 @@ echo "./eessi_container.sh --access rw"
 echo "                     ${CONTAINER_OPT}"
 echo "                     ${HTTP_PROXY_OPT}"
 echo "                     ${HTTPS_PROXY_OPT}"
-echo "                     --info"
+echo "                     --verbose"
 echo "                     --mode run"
 echo "                     ${REPOSITORY_OPT}"
 echo "                     --resume ${BUILD_TMPDIR}"
@@ -202,7 +202,7 @@ echo "                     ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT
                      ${CONTAINER_OPT} \
                      ${HTTP_PROXY_OPT} \
                      ${HTTPS_PROXY_OPT} \
-                     --info \
+                     --verbose \
                      --mode run \
                      ${REPOSITORY_OPT} \
                      --resume ${BUILD_TMPDIR} \

From c0350a9da92f6632c9ba52a57a5a28e03cb9c040 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 22 Feb 2023 23:14:38 +0100
Subject: [PATCH 094/157] improved messages for resume info and adjusted
 parsing in bot/build.sh

---
 bot/build.sh       | 2 +-
 eessi_container.sh | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 77aa2d9077..f9d8c9cede 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -174,7 +174,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
                      ./install_software_layer.sh "$@" 2>&1 | tee -a ${build_outerr}
 
 # determine temporary directory to resume from
-BUILD_TMPDIR=$(grep 'RESUME_FROM_DIR' ${build_outerr} | sed -e "s/^RESUME_FROM_DIR //")
+BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2)
 
 tar_outerr=$(mktemp tar.outerr.XXXX)
 timestamp=$(date +%s)
diff --git a/eessi_container.sh b/eessi_container.sh
index f1c755938c..060b003f33 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -284,7 +284,7 @@ else
     [[ ${VERBOSE} -eq 1 ]] && echo "skipping sanity checks for /tmp"
   fi
   EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX)
-  echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)."
+  echo "Using ${EESSI_HOST_STORAGE} as tmp directory (to resume session add '--resume ${EESSI_HOST_STORAGE}')."
 fi
 
 # if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE}
@@ -565,7 +565,7 @@ if [[ ! -z ${SAVE} ]]; then
     TGZ=${SAVE}
   fi
   tar cf ${TGZ} -C ${EESSI_TMPDIR} .
-  echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')"
+  echo "Saved contents of tmp directory '${EESSI_TMPDIR}' to tarball '${TGZ}' (to resume session add '--resume ${TGZ}')"
 fi
 
 # TODO clean up tmp by default? only retain if another option provided (--retain-tmp)

From ef9a552aa1b0a96a7b13c4319b501e0a79459766 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 22 Feb 2023 23:25:27 +0100
Subject: [PATCH 095/157] delete one of two tmp storage tarballs created by
 bot/build.sh

---
 bot/build.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/bot/build.sh b/bot/build.sh
index f9d8c9cede..5926f005d5 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -209,4 +209,10 @@ echo "                     ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT
                      --save ${PWD}/previous_tmp/tarball_step \
                      ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
 
+# if two tarballs have been generated, only keep the one from tarball step
+NUM_TARBALLS=$(find ${PWD}/previous_tmp -type f -name "*tgz" | wc -l)
+if [[ ${NUM_TARBALLS} -eq 2 ]]; then
+    rm -f previous_tmp/build_step/*.tgz
+fi
+
 exit 0

From 6974e4b80e3a59643d4eac0f229f50f8f547d430 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 24 Feb 2023 13:33:54 +0100
Subject: [PATCH 096/157] add verbose messages on pulling/copying/reusing of
 container image

---
 eessi_container.sh | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 86a510d56e..01c69bdd92 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -320,6 +320,13 @@ if [[ -z ${SINGULARITY_CACHEDIR} ]]; then
 fi
 [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}"
 
+# if VERBOSE is set to 0 (no arg --verbose), add argument '-q'
+if [[ ${VERBOSE} -eq 0 ]]; then
+    RUN_QUIET='-q'
+else
+    RUN_QUIET=''
+fi
+
 # we try our best to make sure that we retain access to the container image in
 # a subsequent session ("best effort" only because pulling or copying operations
 # can fail ... in those cases the script may still succeed, but it is not
@@ -342,7 +349,10 @@ if [[ ${CONTAINER} =~ ${CONTAINER_URL_FMT} ]]; then
     # pull container to ${EESSI_TMPDIR} if it is not there yet (i.e. when
     # resuming from a previous session)
     if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then
-        singularity pull ${EESSI_TMPDIR}/${CONTAINER_IMG} ${CONTAINER}
+        [[ ${VERBOSE} -eq 1 ]] && echo "Pulling container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}"
+        singularity ${RUN_QUIET} pull ${EESSI_TMPDIR}/${CONTAINER_IMG} ${CONTAINER}
+    else
+        [[ ${VERBOSE} -eq 1 ]] && echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}"
     fi
 else
     # determine file name as basename of CONTAINER
@@ -350,7 +360,10 @@ else
     # copy image file to ${EESSI_TMPDIR} if it is not there yet (i.e. when
     # resuming from a previous session)
     if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then
+        [[ ${VERBOSE} -eq 1 ]] && echo "Copying container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}"
         cp -a ${CONTAINER} ${EESSI_TMPDIR}/.
+    else
+        [[ ${VERBOSE} -eq 1 ]] && echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}"
     fi
 fi
 # let CONTAINER point to the pulled, copied or resumed image file
@@ -538,13 +551,6 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
 fi
 
-# if VERBOSE is set to 0 (no arg --verbose), add argument '-q'
-if [[ ${VERBOSE} -eq 0 ]]; then
-    RUN_QUIET='-q'
-else
-    RUN_QUIET=''
-fi
-
 echo "Launching container with command (next line):"
 echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
 singularity ${RUN_QUIET} ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"

From 54bdea52e86d853a923d4884c4b62682440c5b09 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 24 Feb 2023 13:37:47 +0100
Subject: [PATCH 097/157] don't require --verbose for info message on
 pulling/copying/reusing container image

---
 eessi_container.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 01c69bdd92..e98cc53493 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -349,10 +349,10 @@ if [[ ${CONTAINER} =~ ${CONTAINER_URL_FMT} ]]; then
     # pull container to ${EESSI_TMPDIR} if it is not there yet (i.e. when
     # resuming from a previous session)
     if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then
-        [[ ${VERBOSE} -eq 1 ]] && echo "Pulling container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}"
+        echo "Pulling container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}"
         singularity ${RUN_QUIET} pull ${EESSI_TMPDIR}/${CONTAINER_IMG} ${CONTAINER}
     else
-        [[ ${VERBOSE} -eq 1 ]] && echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}"
+        echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}"
     fi
 else
     # determine file name as basename of CONTAINER
@@ -360,10 +360,10 @@ else
     # copy image file to ${EESSI_TMPDIR} if it is not there yet (i.e. when
     # resuming from a previous session)
     if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then
-        [[ ${VERBOSE} -eq 1 ]] && echo "Copying container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}"
+        echo "Copying container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}"
         cp -a ${CONTAINER} ${EESSI_TMPDIR}/.
     else
-        [[ ${VERBOSE} -eq 1 ]] && echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}"
+        echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}"
     fi
 fi
 # let CONTAINER point to the pulled, copied or resumed image file

From a4cea9afb12242340360c9ad922e13c8492c9d57 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 24 Feb 2023 13:53:06 +0100
Subject: [PATCH 098/157] add check to make sure that container image exists in
 tmpdir

---
 .github/workflows/test_eessi_container_script.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index d4a9dfd53e..929fb22cec 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -108,6 +108,9 @@ jobs:
                 tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
                 rm -f ${outfile}
 
+                # make sure that container image exists
+                test -f ${tmpdir}/ghcr.io_eessi_build_node_debian11.sif || (echo "Container image not found in ${tmpdir}" >&2 && ls ${tmpdir} && exit 1)
+
                 ./eessi_container.sh --verbose --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile}
                 cat ${outfile}
                 grep "Resuming from previous run using temporary storage at ${tmpdir}" ${outfile}

From c9758326512d824ae11e790f836c4db36dc3c527 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 24 Feb 2023 15:46:52 +0100
Subject: [PATCH 099/157] fix determining filename from container URL

---
 eessi_container.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index e98cc53493..1d76360735 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -344,8 +344,8 @@ fi
 CONTAINER_IMG=
 CONTAINER_URL_FMT=".*://(.*)"
 if [[ ${CONTAINER} =~ ${CONTAINER_URL_FMT} ]]; then
-    # replace : and - with _ in match (everything after ://) and append .sif
-    CONTAINER_IMG=${BASH_REMATCH[1]//[:-]/_}.sif
+    # replace ':', '-', '/' with '_' in match (everything after ://) and append .sif
+    CONTAINER_IMG="$(echo ${BASH_REMATCH[1]} | sed 's/[:\/-]/_/g').sif"
     # pull container to ${EESSI_TMPDIR} if it is not there yet (i.e. when
     # resuming from a previous session)
     if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then

From 7cc65750c50bae0a2ea3f0cb9d96137fb21aa452 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Sat, 25 Feb 2023 02:55:37 +0100
Subject: [PATCH 100/157] support cmd line arg flag terminator

---
 eessi_container.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/eessi_container.sh b/eessi_container.sh
index b1f8922d61..b9553a7ea1 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -173,6 +173,11 @@ while [[ $# -gt 0 ]]; do
       export https_proxy=${HTTPS_PROXY}
       shift 2
       ;;
+    --)
+      shift
+      POSITIONAL_ARGS+=("$@") # save positional args
+      break
+      ;;
     -*|--*)
       fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}"
       ;;

From 0def4e91ce9d52281e0e8fc3d30c96f095c75bd5 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Fri, 24 Feb 2023 22:07:02 +0100
Subject: [PATCH 101/157] just print contents of cfg/job.cfg

---
 bot/build.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bot/build.sh b/bot/build.sh
index 5926f005d5..906f1bfd7c 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -23,6 +23,10 @@
 # defaults
 export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
 
+echo "bot/build.sh: Showing job.cfg from software-layer side"
+cat cfg/job.cfg
+exit 0
+
 # source utils.sh
 source utils.sh
 

From ef608d15f20bea94bae116b7ece71a21586c7e3c Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Fri, 24 Feb 2023 23:55:44 +0100
Subject: [PATCH 102/157] switching from json (YQ) to ini (cfg_files.sh)

---
 bot/build.sh | 43 ++++++++++++++++---------------------------
 1 file changed, 16 insertions(+), 27 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 906f1bfd7c..c621b9524f 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -17,50 +17,39 @@
 #  - the working directory contains a directory 'cfg' where the main config
 #    file 'job.cfg' has been deposited
 #  - the directory may contain any additional files references in job.cfg
-#  - the tool 'yq' for working with json files is available via the PATH or
-#    the environment variable BOT_YQ (see https://github.com/mikefarah/yq)
 
 # defaults
 export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
 
 echo "bot/build.sh: Showing job.cfg from software-layer side"
 cat cfg/job.cfg
-exit 0
-
-# source utils.sh
-source utils.sh
 
-# check setup / define key variables
-# get path for 'yq' (if not found, an empty string is returned)
-YQ=$(get_path_for_tool "yq" "BOT_YQ")
-exit_code=$?
-if [[ ${exit_code} -ne 0 ]]; then
-    fatal_error "could not find path to 'yq'; exiting"
-else
-    echo_green "found yq (${YQ})"
-fi
+# source utils.sh and cfg_files.sh
+source scripts/utils.sh
+source scripts/cfg_files.sh
 
 # check if './cfg/job.cfg' exists
 if [[ ! -r "${JOB_CFG_FILE}" ]]; then
     fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable"
 fi
 echo "obtaining configuration settings from '${JOB_CFG_FILE}'"
+cfg_load ${JOB_CFG_FILE}
 
 # if http_proxy is in cfg/job.cfg use it, if not use env var $http_proxy
-HTTP_PROXY=$(${YQ} '.site_config.http_proxy // ""' ${JOB_CFG_FILE})
+HTTP_PROXY=$(cfg_get_value "site_config" "http_proxy")
 HTTP_PROXY=${HTTP_PROXY:-${http_proxy}}
 echo "HTTP_PROXY='${HTTP_PROXY}'"
 
 # if https_proxy is in cfg/job.cfg use it, if not use env var $https_proxy
-HTTPS_PROXY=$(${YQ} '.site_config.https_proxy // ""' ${JOB_CFG_FILE})
+HTTPS_PROXY=$(cfg_get_value "site_config" "https_proxy")
 HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}}
 echo "HTTPS_PROXY='${HTTPS_PROXY}'"
 
-LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE})
+LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp")
 echo "LOCAL_TMP='${LOCAL_TMP}'"
 # TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided
 
-SINGULARITY_CACHEDIR=$(${YQ} '.site_config.container_cachedir // ""' ${JOB_CFG_FILE})
+SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir")
 echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'"
 if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then
     export SINGULARITY_CACHEDIR
@@ -73,11 +62,11 @@ STORAGE=$(envsubst <<< ${LOCAL_TMP})
 echo "'${STORAGE}'"
 
 # obtain list of modules to be loaded
-LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' ${JOB_CFG_FILE})
+LOAD_MODULES=$(cfg_get_value "site_config" "load_modules")
 echo "LOAD_MODULES='${LOAD_MODULES}'"
 
 # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND
-CONTAINER=$(${YQ} '.repository.container // ""' ${JOB_CFG_FILE})
+CONTAINER=$(cfg_get_value "repository" "container")
 export SINGULARITY_HOME="$(pwd):/eessi_bot_job"
 export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir"
 mkdir -p ${SINGULARITY_TMPDIR}
@@ -94,20 +83,20 @@ else
 fi
 
 # determine repository to be used from entry .repository in cfg/job.cfg
-REPOSITORY=$(${YQ} '.repository.repo_id // ""' ${JOB_CFG_FILE})
-EESSI_REPOS_CFG_DIR_OVERRIDE=$(${YQ} '.repository.repos_cfg_dir // ""' ${JOB_CFG_FILE})
+REPOSITORY=$(cfg_get_value "repository" "repo_id")
+EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir")
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg}
 
 # determine pilot version to be used from .repository.repo_version in cfg/job.cfg
 # here, just set & export EESSI_PILOT_VERSION_OVERRIDE
 # next script (eessi_container.sh) makes use of it via sourcing init scripts
 # (e.g., init/eessi_defaults or init/minimal_eessi_env)
-export EESSI_PILOT_VERSION_OVERRIDE=$(${YQ} '.repository.repo_version // ""' ${JOB_CFG_FILE})
+export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version")
 
 # determine CVMFS repo to be used from .repository.repo_name in cfg/job.cfg
 # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down
 # "source init/eessi_defaults" via sourcing init/minimal_eessi_env
-export EESSI_CVMFS_REPO_OVERRIDE=$(${YQ} '.repository.repo_name // ""' ${JOB_CFG_FILE})
+export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name")
 
 
 # determine architecture to be used from entry .architecture in cfg/job.cfg
@@ -115,12 +104,12 @@ export EESSI_CVMFS_REPO_OVERRIDE=$(${YQ} '.repository.repo_name // ""' ${JOB_CFG
 if [[ ! -z "${CPU_TARGET}" ]]; then
     EESSI_SOFTWARE_SUBDIR_OVERRIDE=${CPU_TARGET}
 else
-    EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE})
+    EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(cfg_get_value "architecture" "software_subdir")
 fi
 export EESSI_SOFTWARE_SUBDIR_OVERRIDE
 
 # get EESSI_OS_TYPE from .architecture.os_type in cfg/job.cfg (default: linux)
-EESSI_OS_TYPE=$(${YQ} '.architecture.os_type // ""' ${JOB_CFG_FILE})
+EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type")
 export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux}
 
 # TODO

From 429d5a03ad0da75e8e8c52fb8897b341d1c70511 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Thu, 23 Feb 2023 23:56:28 +0100
Subject: [PATCH 103/157] add --generic arg when running install script

---
 bot/build.sh | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index c621b9524f..387a9c95c9 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -140,6 +140,11 @@ REPOSITORY_OPT=
 if [[ ! -z ${REPOSITORY} ]]; then
     REPOSITORY_OPT="--repository ${REPOSITORY}"
 fi
+GENERIC_OPT=
+if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ ".*/generic" ]]; then
+    GENERIC_OPT="--generic"
+fi
+
 mkdir -p previous_tmp/{build_step,tarball_step}
 build_outerr=$(mktemp build.outerr.XXXX)
 echo "Executing command to build software:"
@@ -152,7 +157,7 @@ echo "                     --mode run"
 echo "                     ${REPOSITORY_OPT}"
 echo "                     --save ${PWD}/previous_tmp/build_step"
 echo "                     --storage ${STORAGE}"
-echo "                     ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${build_outerr}"
+echo "                     ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}"
 # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
 ./eessi_container.sh --access rw \
@@ -164,7 +169,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
                      ${REPOSITORY_OPT} \
                      --save ${PWD}/previous_tmp/build_step \
                      --storage ${STORAGE} \
-                     ./install_software_layer.sh "$@" 2>&1 | tee -a ${build_outerr}
+                     ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr}
 
 # determine temporary directory to resume from
 BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2)

From bd31faa6b6e4e3945447118b6121667d01b4af7d Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Fri, 24 Feb 2023 00:17:57 +0100
Subject: [PATCH 104/157] fix error in regex

---
 bot/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/build.sh b/bot/build.sh
index 387a9c95c9..860f888a46 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -141,7 +141,7 @@ if [[ ! -z ${REPOSITORY} ]]; then
     REPOSITORY_OPT="--repository ${REPOSITORY}"
 fi
 GENERIC_OPT=
-if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ ".*/generic" ]]; then
+if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
     GENERIC_OPT="--generic"
 fi
 

From 1bafb7889fa8f49b183e048d63a8fbdabee041f1 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Fri, 24 Feb 2023 00:43:23 +0100
Subject: [PATCH 105/157] use command separator --

---
 bot/build.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 860f888a46..cdd59d6081 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -157,7 +157,7 @@ echo "                     --mode run"
 echo "                     ${REPOSITORY_OPT}"
 echo "                     --save ${PWD}/previous_tmp/build_step"
 echo "                     --storage ${STORAGE}"
-echo "                     ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}"
+echo "                     -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}"
 # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
 ./eessi_container.sh --access rw \
@@ -169,7 +169,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
                      ${REPOSITORY_OPT} \
                      --save ${PWD}/previous_tmp/build_step \
                      --storage ${STORAGE} \
-                     ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr}
+                     -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr}
 
 # determine temporary directory to resume from
 BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2)

From c9041e34604ec881e6a5aeb96fbf31da91bf63a1 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Sat, 25 Feb 2023 00:29:25 +0100
Subject: [PATCH 106/157] try fixing arg issue

---
 bot/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/build.sh b/bot/build.sh
index cdd59d6081..f56f456dcf 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -161,7 +161,7 @@ echo "                     -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\"
 # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
 ./eessi_container.sh --access rw \
-                     ${CONTAINER_OPT} \
+                     "${CONTAINER_OPT}" \
                      ${HTTP_PROXY_OPT} \
                      ${HTTPS_PROXY_OPT} \
                      --verbose \

From 9d46a22892a4462f1b87c52c679d90cfed6256bb Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Sat, 25 Feb 2023 00:41:36 +0100
Subject: [PATCH 107/157] put build args into an array

---
 bot/build.sh | 27 +++++++++++----------------
 1 file changed, 11 insertions(+), 16 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index f56f456dcf..8720c137a5 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -124,21 +124,30 @@ export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux}
 #     files into './cfg/.' and defining '.repository.repos_cfg_dir' in './cfg/job.cfg')
 
 # prepare options and directories for calling eessi_container.sh
+declare -a BUILD_STEP_ARGS=()
+BUILD_STEP_ARGS+=("--access" "rw")
+BUILD_STEP_ARGS+=("--mode" "run")
+BUILD_STEP_ARGS+=("--save" "${PWD}/previous_tmp/build_step")
+BUILD_STEP_ARGS+=("--storage" "${STORAGE}")
 CONTAINER_OPT=
 if [[ ! -z ${CONTAINER} ]]; then
     CONTAINER_OPT="--container ${CONTAINER}"
+    BUILD_STEP_ARGS+=("--container" "${CONTAINER}")
 fi
 HTTP_PROXY_OPT=
 if [[ ! -z ${HTTP_PROXY} ]]; then
     HTTP_PROXY_OPT="--http-proxy ${HTTP_PROXY}"
+    BUILD_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}")
 fi
 HTTPS_PROXY_OPT=
 if [[ ! -z ${HTTPS_PROXY} ]]; then
     HTTPS_PROXY_OPT="--https-proxy ${HTTPS_PROXY}"
+    BUILD_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}")
 fi
 REPOSITORY_OPT=
 if [[ ! -z ${REPOSITORY} ]]; then
     REPOSITORY_OPT="--repository ${REPOSITORY}"
+    BUILD_STEP_ARGS+=("--repository" "${REPOSITORY}")
 fi
 GENERIC_OPT=
 if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
@@ -148,27 +157,13 @@ fi
 mkdir -p previous_tmp/{build_step,tarball_step}
 build_outerr=$(mktemp build.outerr.XXXX)
 echo "Executing command to build software:"
-echo "./eessi_container.sh --access rw"
-echo "                     ${CONTAINER_OPT}"
-echo "                     ${HTTP_PROXY_OPT}"
-echo "                     ${HTTPS_PROXY_OPT}"
+echo "./eessi_container.sh ${BUILD_STEP_ARGS[@]}"
 echo "                     --verbose"
-echo "                     --mode run"
-echo "                     ${REPOSITORY_OPT}"
-echo "                     --save ${PWD}/previous_tmp/build_step"
-echo "                     --storage ${STORAGE}"
 echo "                     -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}"
 # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
-./eessi_container.sh --access rw \
-                     "${CONTAINER_OPT}" \
-                     ${HTTP_PROXY_OPT} \
-                     ${HTTPS_PROXY_OPT} \
+./eessi_container.sh "${BUILD_STEP_ARGS[@]}" \
                      --verbose \
-                     --mode run \
-                     ${REPOSITORY_OPT} \
-                     --save ${PWD}/previous_tmp/build_step \
-                     --storage ${STORAGE} \
                      -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr}
 
 # determine temporary directory to resume from

From ff5045f0940730951c25cfb5260148655fbf933d Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Sat, 25 Feb 2023 01:19:25 +0100
Subject: [PATCH 108/157] use array to define args for build and tarball step

---
 bot/build.sh | 37 +++++++++++++++----------------------
 1 file changed, 15 insertions(+), 22 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 8720c137a5..bb1f4319a1 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -125,29 +125,39 @@ export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux}
 
 # prepare options and directories for calling eessi_container.sh
 declare -a BUILD_STEP_ARGS=()
+BUILD_STEP_ARGS+=("--verbose")
 BUILD_STEP_ARGS+=("--access" "rw")
 BUILD_STEP_ARGS+=("--mode" "run")
 BUILD_STEP_ARGS+=("--save" "${PWD}/previous_tmp/build_step")
 BUILD_STEP_ARGS+=("--storage" "${STORAGE}")
+declare -a TARBALL_STEP_ARGS=()
+TARBALL_STEP_ARGS+=("--verbose")
+TARBALL_STEP_ARGS+=("--access" "rw")
+TARBALL_STEP_ARGS+=("--mode" "run")
+TARBALL_STEP_ARGS+=("--save" "${PWD}/previous_tmp/tarball_step")
 CONTAINER_OPT=
 if [[ ! -z ${CONTAINER} ]]; then
     CONTAINER_OPT="--container ${CONTAINER}"
     BUILD_STEP_ARGS+=("--container" "${CONTAINER}")
+    TARBALL_STEP_ARGS+=("--container" "${CONTAINER}")
 fi
 HTTP_PROXY_OPT=
 if [[ ! -z ${HTTP_PROXY} ]]; then
     HTTP_PROXY_OPT="--http-proxy ${HTTP_PROXY}"
     BUILD_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}")
+    TARBALL_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}")
 fi
 HTTPS_PROXY_OPT=
 if [[ ! -z ${HTTPS_PROXY} ]]; then
     HTTPS_PROXY_OPT="--https-proxy ${HTTPS_PROXY}"
     BUILD_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}")
+    TARBALL_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}")
 fi
 REPOSITORY_OPT=
 if [[ ! -z ${REPOSITORY} ]]; then
     REPOSITORY_OPT="--repository ${REPOSITORY}"
     BUILD_STEP_ARGS+=("--repository" "${REPOSITORY}")
+    TARBALL_STEP_ARGS+=("--repository" "${REPOSITORY}")
 fi
 GENERIC_OPT=
 if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
@@ -158,16 +168,15 @@ mkdir -p previous_tmp/{build_step,tarball_step}
 build_outerr=$(mktemp build.outerr.XXXX)
 echo "Executing command to build software:"
 echo "./eessi_container.sh ${BUILD_STEP_ARGS[@]}"
-echo "                     --verbose"
 echo "                     -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}"
 # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
 ./eessi_container.sh "${BUILD_STEP_ARGS[@]}" \
-                     --verbose \
                      -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr}
 
 # determine temporary directory to resume from
 BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2)
+TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}")
 
 tar_outerr=$(mktemp tar.outerr.XXXX)
 timestamp=$(date +%s)
@@ -181,26 +190,10 @@ export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} $
 # /tmp as default?
 TMP_IN_CONTAINER=/tmp
 echo "Executing command to create tarball:"
-echo "./eessi_container.sh --access rw"
-echo "                     ${CONTAINER_OPT}"
-echo "                     ${HTTP_PROXY_OPT}"
-echo "                     ${HTTPS_PROXY_OPT}"
-echo "                     --verbose"
-echo "                     --mode run"
-echo "                     ${REPOSITORY_OPT}"
-echo "                     --resume ${BUILD_TMPDIR}"
-echo "                     --save ${PWD}/previous_tmp/tarball_step"
-echo "                     ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
-./eessi_container.sh --access rw \
-                     ${CONTAINER_OPT} \
-                     ${HTTP_PROXY_OPT} \
-                     ${HTTPS_PROXY_OPT} \
-                     --verbose \
-                     --mode run \
-                     ${REPOSITORY_OPT} \
-                     --resume ${BUILD_TMPDIR} \
-                     --save ${PWD}/previous_tmp/tarball_step \
-                     ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
+echo "./eessi_container.sh ${TARBALL_STEP_ARGS[@]}"
+echo "                     -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
+./eessi_container.sh "${TARBALL_STEP_ARGS[@]}" \
+                     -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
 
 # if two tarballs have been generated, only keep the one from tarball step
 NUM_TARBALLS=$(find ${PWD}/previous_tmp -type f -name "*tgz" | wc -l)

From d08bd211ed6c23c5b917b01bd01ef75a5cb7b1b9 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Tue, 7 Mar 2023 14:43:52 +0100
Subject: [PATCH 109/157] addressed comments and changes requested by first
 review

---
 bot/build.sh | 172 ++++++++++++++++++++++++---------------------------
 1 file changed, 82 insertions(+), 90 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index bb1f4319a1..004e2d4881 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -16,41 +16,44 @@
 #    pull request (OR by some other means)
 #  - the working directory contains a directory 'cfg' where the main config
 #    file 'job.cfg' has been deposited
-#  - the directory may contain any additional files references in job.cfg
+#  - the directory may contain any additional files referenced in job.cfg
 
-# defaults
-export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
-
-echo "bot/build.sh: Showing job.cfg from software-layer side"
-cat cfg/job.cfg
+# stop as soon as something fails
+set -e
 
 # source utils.sh and cfg_files.sh
 source scripts/utils.sh
 source scripts/cfg_files.sh
 
-# check if './cfg/job.cfg' exists
+# defaults
+export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
+
+# check if ${JOB_CFG_FILE} exists
 if [[ ! -r "${JOB_CFG_FILE}" ]]; then
     fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable"
 fi
-echo "obtaining configuration settings from '${JOB_CFG_FILE}'"
+echo "bot/build.sh: showing ${JOB_CFG_FILE} from software-layer side"
+cat ${JOB_CFG_FILE}
+
+echo "bot/build.sh: obtaining configuration settings from '${JOB_CFG_FILE}'"
 cfg_load ${JOB_CFG_FILE}
 
-# if http_proxy is in cfg/job.cfg use it, if not use env var $http_proxy
+# if http_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $http_proxy
 HTTP_PROXY=$(cfg_get_value "site_config" "http_proxy")
 HTTP_PROXY=${HTTP_PROXY:-${http_proxy}}
-echo "HTTP_PROXY='${HTTP_PROXY}'"
+echo "bot/build.sh: HTTP_PROXY='${HTTP_PROXY}'"
 
-# if https_proxy is in cfg/job.cfg use it, if not use env var $https_proxy
+# if https_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $https_proxy
 HTTPS_PROXY=$(cfg_get_value "site_config" "https_proxy")
 HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}}
-echo "HTTPS_PROXY='${HTTPS_PROXY}'"
+echo "bot/build.sh: HTTPS_PROXY='${HTTPS_PROXY}'"
 
 LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp")
-echo "LOCAL_TMP='${LOCAL_TMP}'"
+echo "bot/build.sh: LOCAL_TMP='${LOCAL_TMP}'"
 # TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided
 
 SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir")
-echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'"
+echo "bot/build.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'"
 if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then
     export SINGULARITY_CACHEDIR
 fi
@@ -61,14 +64,21 @@ echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> "
 STORAGE=$(envsubst <<< ${LOCAL_TMP})
 echo "'${STORAGE}'"
 
+# make sure ${STORAGE} exists
+mkdir -p ${STORAGE}
+
+# make sure the base tmp storage is unique
+JOB_STORAGE=$(mktemp --directory --tmpdir=${STORAGE} bot_job_tmp_XXX)
+echo "bot/build.sh: created unique base tmp storage directory at ${JOB_STORAGE}"
+
 # obtain list of modules to be loaded
 LOAD_MODULES=$(cfg_get_value "site_config" "load_modules")
-echo "LOAD_MODULES='${LOAD_MODULES}'"
+echo "bot/build.sh: LOAD_MODULES='${LOAD_MODULES}'"
 
 # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND
 CONTAINER=$(cfg_get_value "repository" "container")
-export SINGULARITY_HOME="$(pwd):/eessi_bot_job"
-export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir"
+export SINGULARITY_HOME="${pwd}:/eessi_bot_job"
+export SINGULARITY_TMPDIR="${pwd}/singularity_tmpdir"
 mkdir -p ${SINGULARITY_TMPDIR}
 
 # load modules if LOAD_MODULES is not empty
@@ -82,103 +92,91 @@ else
     echo "bot/build.sh: no modules to be loaded"
 fi
 
-# determine repository to be used from entry .repository in cfg/job.cfg
+# determine repository to be used from entry .repository in ${JOB_CFG_FILE}
 REPOSITORY=$(cfg_get_value "repository" "repo_id")
 EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir")
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg}
+echo "bot/build.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'"
 
-# determine pilot version to be used from .repository.repo_version in cfg/job.cfg
+# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE}
 # here, just set & export EESSI_PILOT_VERSION_OVERRIDE
 # next script (eessi_container.sh) makes use of it via sourcing init scripts
 # (e.g., init/eessi_defaults or init/minimal_eessi_env)
 export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version")
+echo "bot/build.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'"
 
-# determine CVMFS repo to be used from .repository.repo_name in cfg/job.cfg
+# determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE}
 # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down
 # "source init/eessi_defaults" via sourcing init/minimal_eessi_env
 export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name")
-
-
-# determine architecture to be used from entry .architecture in cfg/job.cfg
-# default: leave empty to let downstream script(s) determine subdir to be used
-if [[ ! -z "${CPU_TARGET}" ]]; then
-    EESSI_SOFTWARE_SUBDIR_OVERRIDE=${CPU_TARGET}
-else
-    EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(cfg_get_value "architecture" "software_subdir")
-fi
+echo "bot/build.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'"
+
+# determine architecture to be used from entry .architecture in ${JOB_CFG_FILE}
+# fallbacks:
+#  - ${CPU_TARGET} handed over from bot
+#  - left empty to let downstream script(s) determine subdir to be used
+EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(cfg_get_value "architecture" "software_subdir")
+EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE:-${CPU_TARGET}}
 export EESSI_SOFTWARE_SUBDIR_OVERRIDE
+echo "bot/build.sh: EESSI_SOFTWARE_SUBDIR_OVERRIDE='${EESSI_SOFTWARE_SUBDIR_OVERRIDE}'"
 
-# get EESSI_OS_TYPE from .architecture.os_type in cfg/job.cfg (default: linux)
+# get EESSI_OS_TYPE from .architecture.os_type in ${JOB_CFG_FILE} (default: linux)
 EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type")
 export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux}
-
-# TODO
-#   - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh
-#     TODO ensure that the bot makes use of that. (currently sets env var
-#     CPU_TARGET & adds --export=ALL,CPU_TARGET=val to sbatch command ... also
-#     add it to cfg/job.cfg - .architecture.software_subdir)
-#   - CODED add handling of http(s)_proxy to eessi_container.sh, in there needs the
-#     CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust
-#     way to determine the IP address of a proxy)
-#   - bot needs to make repos.cfg and cfg_bundle available to job (likely, by copying
-#     files into './cfg/.' and defining '.repository.repos_cfg_dir' in './cfg/job.cfg')
-
-# prepare options and directories for calling eessi_container.sh
+echo "bot/build.sh: EESSI_OS_TYPE='${EESSI_OS_TYPE}'"
+
+# prepare arguments to eessi_container.sh common to build and tarball steps
+declare -a COMMON_ARGS=()
+COMMON_ARGS+=("--verbose")
+COMMON_ARGS+=("--access" "rw")
+COMMON_ARGS+=("--mode" "run")
+[[ ! -z ${CONTAINER} ]] && COMMON_ARGS+=("--container" "${CONTAINER}")
+[[ ! -z ${HTTP_PROXY} ]] && COMMON_ARGS+=("--http-proxy" "${HTTP_PROXY}")
+[[ ! -z ${HTTPS_PROXY} ]] && COMMON_ARGS+=("--https-proxy" "${HTTPS_PROXY}")
+[[ ! -z ${REPOSITORY} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY}")
+
+# make sure to use the same parent dir for storing tarballs of tmp
+PREVIOUS_TMP_DIR=${PWD}/previous_tmp
+
+# prepare directory to store tarball of tmp for build step
+TARBALL_TMP_BUILD_STEP_DIR=${PREVIOUS_TMP_DIR}/build_step
+mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR}
+
+# prepare arguments to eessi_container.sh specific to build step
 declare -a BUILD_STEP_ARGS=()
-BUILD_STEP_ARGS+=("--verbose")
-BUILD_STEP_ARGS+=("--access" "rw")
-BUILD_STEP_ARGS+=("--mode" "run")
-BUILD_STEP_ARGS+=("--save" "${PWD}/previous_tmp/build_step")
+BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}")
 BUILD_STEP_ARGS+=("--storage" "${STORAGE}")
-declare -a TARBALL_STEP_ARGS=()
-TARBALL_STEP_ARGS+=("--verbose")
-TARBALL_STEP_ARGS+=("--access" "rw")
-TARBALL_STEP_ARGS+=("--mode" "run")
-TARBALL_STEP_ARGS+=("--save" "${PWD}/previous_tmp/tarball_step")
-CONTAINER_OPT=
-if [[ ! -z ${CONTAINER} ]]; then
-    CONTAINER_OPT="--container ${CONTAINER}"
-    BUILD_STEP_ARGS+=("--container" "${CONTAINER}")
-    TARBALL_STEP_ARGS+=("--container" "${CONTAINER}")
-fi
-HTTP_PROXY_OPT=
-if [[ ! -z ${HTTP_PROXY} ]]; then
-    HTTP_PROXY_OPT="--http-proxy ${HTTP_PROXY}"
-    BUILD_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}")
-    TARBALL_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}")
-fi
-HTTPS_PROXY_OPT=
-if [[ ! -z ${HTTPS_PROXY} ]]; then
-    HTTPS_PROXY_OPT="--https-proxy ${HTTPS_PROXY}"
-    BUILD_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}")
-    TARBALL_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}")
-fi
-REPOSITORY_OPT=
-if [[ ! -z ${REPOSITORY} ]]; then
-    REPOSITORY_OPT="--repository ${REPOSITORY}"
-    BUILD_STEP_ARGS+=("--repository" "${REPOSITORY}")
-    TARBALL_STEP_ARGS+=("--repository" "${REPOSITORY}")
-fi
+
+# prepare arguments to install_software_layer.sh (specific to build step)
 GENERIC_OPT=
 if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
     GENERIC_OPT="--generic"
 fi
 
-mkdir -p previous_tmp/{build_step,tarball_step}
+# create tmp file for output of build step
 build_outerr=$(mktemp build.outerr.XXXX)
+
 echo "Executing command to build software:"
-echo "./eessi_container.sh ${BUILD_STEP_ARGS[@]}"
+echo "./eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}"
 echo "                     -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}"
-# set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
-export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
-./eessi_container.sh "${BUILD_STEP_ARGS[@]}" \
+./eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \
                      -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr}
 
+# prepare directory to store tarball of tmp for tarball step
+TARBALL_TMP_TARBALL_STEP_DIR=${PREVIOUS_TMP_DIR}/tarball_step
+mkdir -p ${TARBALL_TMP_TARBALL_STEP_DIR}
+
+# create tmp file for output of tarball step
+tar_outerr=$(mktemp tar.outerr.XXXX)
+
+# prepare arguments to eessi_container.sh specific to tarball step
+declare -a TARBALL_STEP_ARGS=()
+TARBALL_STEP_ARGS+=("--save" "${TARBALL_TMP_TARBALL_STEP_DIR}")
+
 # determine temporary directory to resume from
 BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2)
 TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}")
 
-tar_outerr=$(mktemp tar.outerr.XXXX)
 timestamp=$(date +%s)
 # to set EESSI_PILOT_VERSION we need to source init/eessi_defaults now
 source init/eessi_defaults
@@ -190,15 +188,9 @@ export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} $
 # /tmp as default?
 TMP_IN_CONTAINER=/tmp
 echo "Executing command to create tarball:"
-echo "./eessi_container.sh ${TARBALL_STEP_ARGS[@]}"
+echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}"
 echo "                     -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
-./eessi_container.sh "${TARBALL_STEP_ARGS[@]}" \
+./eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \
                      -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
 
-# if two tarballs have been generated, only keep the one from tarball step
-NUM_TARBALLS=$(find ${PWD}/previous_tmp -type f -name "*tgz" | wc -l)
-if [[ ${NUM_TARBALLS} -eq 2 ]]; then
-    rm -f previous_tmp/build_step/*.tgz
-fi
-
 exit 0

From f59bfdededf7b56bc95dc64f19021a3770d8ac61 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Tue, 7 Mar 2023 15:33:24 +0100
Subject: [PATCH 110/157] fix check for missing installations

---
 check_missing_installations.sh | 20 ++++++++++++++++++--
 scripts/utils.sh               |  2 +-
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/check_missing_installations.sh b/check_missing_installations.sh
index e4c75aebd8..45f276dee1 100755
--- a/check_missing_installations.sh
+++ b/check_missing_installations.sh
@@ -27,5 +27,21 @@ eb_missing_out=$LOCAL_TMPDIR/eb_missing.out
 # we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used
 # PR #16531: Nextflow-22.10.1.eb
 ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out}
-grep "No missing modules" ${eb_missing_out} > /dev/null
-check_exit_code $? "${ok_msg}" "${fail_msg}"
+
+# the above assesses the installed software for each easyconfig provided in
+# the easystack file and then print messages such as
+# `No missing modules!`
+# or
+# `2 out of 3 required modules missing:`
+# depending on the result of the assessment. Hence, we need to check if the
+# output does not contain any line with ` required modules missing:`
+
+grep " required modules missing:" ${eb_missing_out} > /dev/null
+
+# we need to process the result (from finding `No missing modules` to NOT finding
+# ` required modules missing:` and no other error happened)
+#
+# if grep returns 1 (` required modules missing:` was NOT found), we set
+# MODULES_MISSING to 0, otherwise (it was found or another error) we set it to 1
+[[ $? -eq 1 ]] && MODULES_MISSING=0 || MODULES_MISSING=1
+check_exit_code ${MODULES_MISSING} "${ok_msg}" "${fail_msg}"
diff --git a/scripts/utils.sh b/scripts/utils.sh
index d0da95e87f..1f46ae69b4 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -28,7 +28,7 @@ function check_exit_code {
     if [[ $ec -eq 0 ]]; then
         echo_green "${ok_msg}"
     else
-        fatal_error "${fail_msg}"
+        echo_red "${fail_msg}"
     fi
 }
 

From 7bb7c1de90b2bfb47a11124702d6dd968531267a Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 8 Mar 2023 11:07:04 +0100
Subject: [PATCH 111/157] add test with missing package

---
 .github/workflows/test_eessi.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
index 92b1f71cad..1735bb77ed 100644
--- a/.github/workflows/test_eessi.yml
+++ b/.github/workflows/test_eessi.yml
@@ -40,4 +40,13 @@ jobs:
               export EESSI_OS_TYPE=linux
               export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
+              echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
               ./check_missing_installations.sh
+              echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package"
+              echo "  GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml
+              echo "    toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml
+              echo "      SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml
+              echo "        versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml
+              tail -n 8 eessi-${{matrix.EESSI_VERSION}}.yml
+              ./check_missing_installations.sh
+              [[ $? -eq 0 ]] && exit 1 || exit 0

From 68a068b64ea5af8fad5a69ff812a5942cacd7167 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 8 Mar 2023 11:25:04 +0100
Subject: [PATCH 112/157] separated test cases + updated evaluation of test
 result

---
 .github/workflows/test_eessi.yml | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
index 1735bb77ed..073edaf4e8 100644
--- a/.github/workflows/test_eessi.yml
+++ b/.github/workflows/test_eessi.yml
@@ -42,11 +42,26 @@ jobs:
               env | grep ^EESSI | sort
               echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
               ./check_missing_installations.sh
+
+        - name: Test check_missing_installations.sh with missing package
+          run: |
+              source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
+              module load EasyBuild
+              eb --version
+              export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
+              export EESSI_OS_TYPE=linux
+              export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
+              env | grep ^EESSI | sort
               echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package"
               echo "  GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml
               echo "    toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml
               echo "      SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml
               echo "        versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml
               tail -n 8 eessi-${{matrix.EESSI_VERSION}}.yml
-              ./check_missing_installations.sh
-              [[ $? -eq 0 ]] && exit 1 || exit 0
+              if ./check_missing_installations.sh; then
+                echo "captured missing package; test PASSED"
+                exit 0
+              else
+                echo "did NOT capture missing package; test FAILED"
+                exit 1
+              fi

From 72185ba91dde02c1852d5b59ab690ccb0d5d1ceb Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 8 Mar 2023 11:30:17 +0100
Subject: [PATCH 113/157] fix small bug in env var name

---
 .github/workflows/test_eessi.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
index 073edaf4e8..f3bf2e7ab1 100644
--- a/.github/workflows/test_eessi.yml
+++ b/.github/workflows/test_eessi.yml
@@ -38,7 +38,7 @@ jobs:
               eb --version
               export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
               export EESSI_OS_TYPE=linux
-              export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
+              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
               echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
               ./check_missing_installations.sh
@@ -50,7 +50,7 @@ jobs:
               eb --version
               export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
               export EESSI_OS_TYPE=linux
-              export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
+              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
               echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package"
               echo "  GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml

From 094e82df3b0de5a195d99fa914674024aa2e8145 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 8 Mar 2023 11:35:29 +0100
Subject: [PATCH 114/157] set software subdir override before sourcing init
 script

---
 .github/workflows/test_eessi.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
index f3bf2e7ab1..d0b530e2e4 100644
--- a/.github/workflows/test_eessi.yml
+++ b/.github/workflows/test_eessi.yml
@@ -33,24 +33,24 @@ jobs:
 
         - name: Test check_missing_installations.sh script
           run: |
+              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
               module load EasyBuild
               eb --version
               export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
               export EESSI_OS_TYPE=linux
-              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
               echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
               ./check_missing_installations.sh
 
         - name: Test check_missing_installations.sh with missing package
           run: |
+              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
               module load EasyBuild
               eb --version
               export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
               export EESSI_OS_TYPE=linux
-              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
               echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package"
               echo "  GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml

From 60cf7e2715b7231810479a3fd91d20499c9b7670 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 8 Mar 2023 11:42:01 +0100
Subject: [PATCH 115/157] Revert "set software subdir override before sourcing
 init script"

This reverts commit 094e82df3b0de5a195d99fa914674024aa2e8145.
---
 .github/workflows/test_eessi.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
index d0b530e2e4..f3bf2e7ab1 100644
--- a/.github/workflows/test_eessi.yml
+++ b/.github/workflows/test_eessi.yml
@@ -33,24 +33,24 @@ jobs:
 
         - name: Test check_missing_installations.sh script
           run: |
-              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
               module load EasyBuild
               eb --version
               export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
               export EESSI_OS_TYPE=linux
+              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
               echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
               ./check_missing_installations.sh
 
         - name: Test check_missing_installations.sh with missing package
           run: |
-              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
               module load EasyBuild
               eb --version
               export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
               export EESSI_OS_TYPE=linux
+              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
               echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package"
               echo "  GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml

From 6f471520b63b0cee6b9d94818e7f42938dbc1bf8 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 8 Mar 2023 11:42:51 +0100
Subject: [PATCH 116/157] Revert "fix small bug in env var name"

This reverts commit 72185ba91dde02c1852d5b59ab690ccb0d5d1ceb.
---
 .github/workflows/test_eessi.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
index f3bf2e7ab1..073edaf4e8 100644
--- a/.github/workflows/test_eessi.yml
+++ b/.github/workflows/test_eessi.yml
@@ -38,7 +38,7 @@ jobs:
               eb --version
               export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
               export EESSI_OS_TYPE=linux
-              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
+              export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
               echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
               ./check_missing_installations.sh
@@ -50,7 +50,7 @@ jobs:
               eb --version
               export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
               export EESSI_OS_TYPE=linux
-              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
+              export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
               echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package"
               echo "  GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml

From a15676ddac70b5cb18d82625a9a048d59199a41b Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 8 Mar 2023 11:48:13 +0100
Subject: [PATCH 117/157] tweaking output of test

---
 .github/workflows/test_eessi.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
index 073edaf4e8..23b30c0063 100644
--- a/.github/workflows/test_eessi.yml
+++ b/.github/workflows/test_eessi.yml
@@ -43,7 +43,7 @@ jobs:
               echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
               ./check_missing_installations.sh
 
-        - name: Test check_missing_installations.sh with missing package
+        - name: Test check_missing_installations.sh with missing package (GCC/8.3.0)
           run: |
               source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
               module load EasyBuild
@@ -52,12 +52,12 @@ jobs:
               export EESSI_OS_TYPE=linux
               export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
-              echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package"
+              echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package (GCC/8.3.0)"
               echo "  GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml
               echo "    toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml
               echo "      SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml
               echo "        versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml
-              tail -n 8 eessi-${{matrix.EESSI_VERSION}}.yml
+              tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml
               if ./check_missing_installations.sh; then
                 echo "captured missing package; test PASSED"
                 exit 0

From 54d2a21f04f7e10ee8577c674be7b2d411da5026 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 8 Mar 2023 13:31:25 +0100
Subject: [PATCH 118/157] improved usage information

---
 eessi_container.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index b9553a7ea1..48c4653ba9 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -67,7 +67,7 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR}/repos.cfg"
 #    https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash
 
 display_help() {
-  echo "usage: $0 [OPTIONS] [SCRIPT]"
+  echo "usage: $0 [OPTIONS] [[--] SCRIPT or COMMAND]"
   echo " OPTIONS:"
   echo "  -a | --access {ro,rw}  - ro (read-only), rw (read & write) [default: ro]"
   echo "  -c | --container IMG   - image file or URL defining the container to use"
@@ -77,7 +77,7 @@ display_help() {
   echo "                           temporary data) [default: 1. TMPDIR, 2. /tmp]"
   echo "  -l | --list-repos      - list available repository identifiers [default: false]"
   echo "  -m | --mode MODE       - with MODE==shell (launch interactive shell) or"
-  echo "                           MODE==run (run a script) [default: shell]"
+  echo "                           MODE==run (run a script or command) [default: shell]"
   echo "  -r | --repository CFG  - configuration file or identifier defining the"
   echo "                           repository to use [default: EESSI-pilot via"
   echo "                           default container, see --container]"
@@ -98,7 +98,9 @@ display_help() {
   echo "  -y | --https-proxy URL - provides URL for the env variable https_proxy"
   echo "                           [default: not set]; uses env var \$https_proxy if set"
   echo
-  echo " If value for --mode is 'run', the SCRIPT provided is executed."
+  echo " If value for --mode is 'run', the SCRIPT/COMMAND provided is executed. If"
+  echo " arguments to the script/command start with '-' or '--', use the flag terminator"
+  echo " '--' to let eessi_container.sh stop parsing arguments."
 }
 
 # set defaults for command line arguments

From 99231e84eade092ae45fb5fa725d752eb04edf92 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Sat, 11 Mar 2023 00:07:09 +0100
Subject: [PATCH 119/157] if exit_code is not zero, use fatal_error (as it was)

---
 scripts/utils.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/utils.sh b/scripts/utils.sh
index 1f46ae69b4..d0da95e87f 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -28,7 +28,7 @@ function check_exit_code {
     if [[ $ec -eq 0 ]]; then
         echo_green "${ok_msg}"
     else
-        echo_red "${fail_msg}"
+        fatal_error "${fail_msg}"
     fi
 }
 

From b94bd111a3dcf422e7e78fa58bdbf52a74415fb2 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Sat, 11 Mar 2023 00:52:43 +0100
Subject: [PATCH 120/157] fix bug in test logic

---
 .github/workflows/test_eessi.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
index 23b30c0063..ce58fc3a97 100644
--- a/.github/workflows/test_eessi.yml
+++ b/.github/workflows/test_eessi.yml
@@ -59,9 +59,9 @@ jobs:
               echo "        versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml
               tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml
               if ./check_missing_installations.sh; then
-                echo "captured missing package; test PASSED"
-                exit 0
-              else
                 echo "did NOT capture missing package; test FAILED"
                 exit 1
+              else
+                echo "captured missing package; test PASSED"
+                exit 0
               fi

From 5e4bea9b71a37a09359def990fd12a1c01a5bf1c Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Sat, 11 Mar 2023 01:03:20 +0100
Subject: [PATCH 121/157] added explanation for test cases (exit code vs
 boolean logic)

---
 .github/workflows/test_eessi.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
index ce58fc3a97..04195dd619 100644
--- a/.github/workflows/test_eessi.yml
+++ b/.github/workflows/test_eessi.yml
@@ -58,6 +58,11 @@ jobs:
               echo "      SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml
               echo "        versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml
               tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml
+              # note, check_missing_installations.sh exits 1 if a package was
+              # missing, which is intepreted as false (exit code based, not
+              # boolean logic), hence when the script exits 0 if no package was
+              # missing it is interpreted as true, thus the test did not capture
+              # the missing package
               if ./check_missing_installations.sh; then
                 echo "did NOT capture missing package; test FAILED"
                 exit 1

From 64d1866638288e315de91a41ad37a40fdaf53736 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Sat, 11 Mar 2023 01:33:50 +0100
Subject: [PATCH 122/157] fix bug, env var PWD is uppercase

---
 bot/build.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 004e2d4881..20334501ed 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -77,8 +77,8 @@ echo "bot/build.sh: LOAD_MODULES='${LOAD_MODULES}'"
 
 # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND
 CONTAINER=$(cfg_get_value "repository" "container")
-export SINGULARITY_HOME="${pwd}:/eessi_bot_job"
-export SINGULARITY_TMPDIR="${pwd}/singularity_tmpdir"
+export SINGULARITY_HOME="${PWD}:/eessi_bot_job"
+export SINGULARITY_TMPDIR="${PWD}/singularity_tmpdir"
 mkdir -p ${SINGULARITY_TMPDIR}
 
 # load modules if LOAD_MODULES is not empty

From ff78c62df541075ecfb58c37e8a14bc4a0e0d80f Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Wed, 15 Mar 2023 07:04:02 +0100
Subject: [PATCH 123/157] make sure that CPU arch specific directories are used
 as container cache dir

---
 bot/build.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bot/build.sh b/bot/build.sh
index 20334501ed..c8def2cdd3 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -27,6 +27,7 @@ source scripts/cfg_files.sh
 
 # defaults
 export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
+HOST_ARCH=$(uname -m)
 
 # check if ${JOB_CFG_FILE} exists
 if [[ ! -r "${JOB_CFG_FILE}" ]]; then
@@ -55,6 +56,8 @@ echo "bot/build.sh: LOCAL_TMP='${LOCAL_TMP}'"
 SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir")
 echo "bot/build.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'"
 if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then
+    # make sure that separate directories are used for different CPU families
+    SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}/${HOST_ARCH}
     export SINGULARITY_CACHEDIR
 fi
 

From 22661e66deb8e31ba053aae3125ca30ca8411f73 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 17 Mar 2023 18:58:28 +0100
Subject: [PATCH 124/157] restore PATHs only after last run of pip installed eb

---
 EESSI-pilot-install-software.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
index 2830754b29..c05ccf8ab9 100755
--- a/EESSI-pilot-install-software.sh
+++ b/EESSI-pilot-install-software.sh
@@ -165,10 +165,6 @@ else
     eb --install-latest-eb-release &> ${eb_install_out}
     check_exit_code $? "${ok_msg}" "${fail_msg}"
 
-    # restore origin $PATH and $PYTHONPATH values
-    export PATH=${ORIG_PATH}
-    export PYTHONPATH=${ORIG_PYTHONPATH}
-
     eb --search EasyBuild-${REQ_EB_VERSION}.eb | grep EasyBuild-${REQ_EB_VERSION}.eb > /dev/null
     if [[ $? -eq 0 ]]; then
         ok_msg="EasyBuild v${REQ_EB_VERSION} installed, alright!"
@@ -177,6 +173,10 @@ else
         check_exit_code $? "${ok_msg}" "${fail_msg}"
     fi
 
+    # restore origin $PATH and $PYTHONPATH values
+    export PATH=${ORIG_PATH}
+    export PYTHONPATH=${ORIG_PYTHONPATH}
+
     module avail easybuild/${REQ_EB_VERSION} &> ${ml_av_easybuild_out}
     if [[ $? -eq 0 ]]; then
         echo_green ">> EasyBuild module installed!"

From 092cb49cf76735df48110fe86a1e9bb4e2c571c0 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 17 Mar 2023 12:42:05 +0100
Subject: [PATCH 125/157] attempt to fix issue #225

---
 create_tarball.sh | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/create_tarball.sh b/create_tarball.sh
index 56ac8ab7ad..08dc4f4d7e 100755
--- a/create_tarball.sh
+++ b/create_tarball.sh
@@ -38,6 +38,7 @@ cd ${overlay_upper_dir}/versions/
 echo ">> Collecting list of files/directories to include in tarball via ${PWD}..."
 
 files_list=${tmpdir}/files.list.txt
+module_files_list=${tmpdir}/module_files.list.txt
 
 if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then
     # include Lmod cache and configuration file (lmodrc.lua),
@@ -49,12 +50,26 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then
     find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list}
     # module symlinks
     find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list}
+    # module files and symlinks
+    find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f -o -type l \
+        | grep -v '/\.wh\.' | sed -e 's/.lua$//' | awk -F'/' '{printf "%s/%s\n", $(NF-1), $NF}' | sort | uniq \
+        >> ${module_files_list}
 fi
 if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software ]; then
     # installation directories
-    ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/*/* | grep -v '/\.wh\.' >> ${files_list}
+    # ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/*/* | grep -v '/\.wh\.' >> ${files_list}
+    for package_version in $(cat ${module_files_list}); do
+        echo "handling ${package_version}"
+        ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \
+            | grep -v '/\.wh\.' >> ${files_list}
+    done
 fi
 
+echo "wrote file list to ${files_list}"
+cat ${files_list}
+echo "wrote module file list to ${module_files_list}"
+cat ${module_files_list}
+
 topdir=${cvmfs_repo}/versions/
 
 echo ">> Creating tarball ${target_tgz} from ${topdir}..."

From 7a750d508d00821cd5b90e38b1471b6a2433a77e Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 17 Mar 2023 19:17:02 +0100
Subject: [PATCH 126/157] add comments, small improvements

---
 create_tarball.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/create_tarball.sh b/create_tarball.sh
index 08dc4f4d7e..9381407d10 100755
--- a/create_tarball.sh
+++ b/create_tarball.sh
@@ -51,13 +51,12 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then
     # module symlinks
     find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list}
     # module files and symlinks
-    find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f -o -type l \
+    find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules/all -type f -o -type l \
         | grep -v '/\.wh\.' | sed -e 's/.lua$//' | awk -F'/' '{printf "%s/%s\n", $(NF-1), $NF}' | sort | uniq \
         >> ${module_files_list}
 fi
 if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software ]; then
-    # installation directories
-    # ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/*/* | grep -v '/\.wh\.' >> ${files_list}
+    # installation directories but only those for which module files were created
     for package_version in $(cat ${module_files_list}); do
         echo "handling ${package_version}"
         ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \
@@ -65,6 +64,7 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software ]; then
     done
 fi
 
+# add a bit debug output
 echo "wrote file list to ${files_list}"
 cat ${files_list}
 echo "wrote module file list to ${module_files_list}"

From 6299c9b4a84a5ced2afde69e9c668c4de9752331 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 17 Mar 2023 19:36:00 +0100
Subject: [PATCH 127/157] add configuration file for production
 build-and-deploy bot in AWS CitC cluster

---
 bot/bot-eessi-aws-citc.cfg | 151 +++++++++++++++++++++++++++++++++++++
 1 file changed, 151 insertions(+)
 create mode 100644 bot/bot-eessi-aws-citc.cfg

diff --git a/bot/bot-eessi-aws-citc.cfg b/bot/bot-eessi-aws-citc.cfg
new file mode 100644
index 0000000000..5b3ad34612
--- /dev/null
+++ b/bot/bot-eessi-aws-citc.cfg
@@ -0,0 +1,151 @@
+# Also see documentation at https://github.com/EESSI/eessi-bot-software-layer/blob/main/README.md#step5.5
+
+[github]
+# replace '123456' with the ID of your GitHub App
+app_id = 281041
+
+# a short (!) name for your app instance that can be used for example
+#   when adding/updating a comment to a PR
+# (!) a short yet descriptive name is preferred because it appears in
+#   comments to the PR
+# for example, the name could include the name of the cluster the bot
+#   runs on and the username which runs the bot
+# NOTE avoid putting an actual username here as it will be visible on
+#      potentially publicly accessible GitHub pages.
+app_name = eessi-bot-citc-aws
+
+# replace '12345678' with the ID of the installation of your GitHub App
+#   (can be derived by creating an event and then checking for the list
+#   of sent events and its payload either via the Smee channel's web page
+#   or via the Advanced section of your GitHub App on github.com)
+installation_id = 33078935
+
+# path to the private key that was generated when the GitHub App was registered
+private_key = /mnt/shared/home/bot/eessi-bot-software-layer/eessi-bot-citc-aws-private-key.pem
+
+
+[buildenv]
+# name of the job script used for building an EESSI stack
+build_job_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/bot-build.slurm
+
+# The container_cachedir may be used to reuse downloaded container image files
+# across jobs. Thus, jobs can more quickly launch containers.
+container_cachedir = /mnt/shared/home/bot/eessi-bot-software-layer/containers-cache-dir
+
+# it may happen that we need to customize some CVMFS configuration
+#   the value of cvmfs_customizations is a dictionary which maps a file
+#   name to an entry that needs to be added to that file
+cvmfs_customizations = {}
+
+# if compute nodes have no internet connection, we need to set http(s)_proxy
+#   or commands such as pip3 cannot download software from package repositories
+#   for example, the temporary EasyBuild is installed via pip3 first
+# http_proxy = http://PROXY_DNS:3128/
+# https_proxy = http://PROXY_DNS:3128/
+
+# directory under which the bot prepares directories per job
+#   structure created is as follows: YYYY.MM/pr_PR_NUMBER/event_EVENT_ID/run_RUN_NUMBER/OS+SUBDIR
+jobs_base_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs
+
+# configure environment
+#   list of comma-separated modules to be loaded by build_job_script
+#   useful/needed if some tool is not provided as system-wide package
+#   (read by bot and handed over to build_job_script via parameter
+#   --load-modules)
+load_modules =
+
+# PATH to temporary directory on build node ... ends up being used for
+#     for example, EESSI_TMPDIR --> /tmp/$USER/EESSI
+#   escaping variables with '\' delays expansion to the start of the
+#     build_job_script; this can be used for referencing environment
+#     variables that are only set inside a Slurm job
+local_tmp = /tmp/$USER/EESSI
+
+# parameters to be added to all job submissions
+# NOTE do not quote parameter string. Quotes are retained when reading in config and
+#      then the whole 'string' is recognised as a single parameter.
+# NOTE 2 '--get-user-env' may be needed on systems where the job's environment needs
+#        to be initialised as if it is for a login shell.
+# note: hardcoded 24h time limit until https://github.com/EESSI/eessi-bot-software-layer/issues/146 is fixed
+slurm_params = --hold --time=24:0:0
+
+# full path to the job submission command
+submit_command = /usr/bin/sbatch
+
+# which GH account has the permission to trigger the build (by setting
+# the label 'bot:build' (apparently this cannot be restricted on GitHub)
+# if value is left/empty everyone can trigger the build
+# value can be a space delimited list of GH accounts
+build_permission = boegel trz42 bedroge
+
+[architecturetargets]
+# defines both for which architectures the bot will build
+#   and what submission parameters shall be used
+# medium instances (8 cores, 16GB RAM)
+#arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge"}
+# larger instances (16 cores, 32GB RAM)
+arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge"}
+
+[repo_targets]
+# defines for which repository a arch_target should be build for
+#
+# only building for repository EESSI-pilot
+repo_target_map = { "linux/x86_64/generic" : ["EESSI-pilot"], "linux/x86_64/intel/haswell" : ["EESSI-pilot"], "linux/x86_64/intel/skylake_avx512" : ["EESSI-pilot"], "linux/x86_64/amd/zen2": ["EESSI-pilot"], "linux/x86_64/amd/zen3" : ["EESSI-pilot"], "linux/aarch64/generic" : ["EESSI-pilot"], "linux/aarch64/graviton2" : ["EESSI-pilot"], "linux/aarch64/graviton3" : ["EESSI-pilot"]}
+
+# points to definition of repositories (default EESSI-pilot defined by build container)
+repos_cfg_dir = /mnt/shared/home/bot/eessi-bot-software-layer/cfg-bundles
+
+# configuration for event handler which receives events from a GitHub repository.
+[event_handler]
+# path to the log file to log messages for event handler
+log_path = /mnt/shared/home/bot/eessi-bot-software-layer/eessi_bot_event_handler.log
+
+
+[job_manager]
+# path to the log file to log messages for job manager
+log_path = /mnt/shared/home/bot/eessi-bot-software-layer/eessi_bot_job_manager.log
+
+# directory where job manager stores information about jobs to be tracked
+#   e.g. as symbolic link JOBID -> directory to job
+job_ids_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs
+
+# full path to the job status checking command
+poll_command = /usr/bin/squeue
+
+# polling interval in seconds
+poll_interval = 60
+
+# full path to the command for manipulating existing jobs
+scontrol_command = /usr/bin/scontrol
+
+[deploycfg]
+# script for uploading built software packages
+tarball_upload_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/eessi-upload-to-staging
+
+# URL to S3/minio bucket
+#   if attribute is set, bucket_base will be constructed as follows
+#     bucket_base=${endpoint_url}/${bucket_name}
+#   otherwise, bucket_base will be constructed as follows
+#     bucket_base=https://${bucket_name}.s3.amazonaws.com
+# - The former variant is used for non AWS S3 services, eg, minio, or when
+#   the bucket name is not provided in the hostname (see latter case).
+# - The latter variant is used for AWS S3 services.
+#endpoint_url = URL_TO_S3_SERVER
+
+# bucket name
+bucket_name = eessi-staging
+
+# upload policy: defines what policy is used for uploading built artefacts
+#                to an S3 bucket
+# 'all' ..: upload all artefacts (mulitple uploads of the same artefact possible)
+# 'latest': for each build target (eessi-VERSION-{software,init,compat}-OS-ARCH)
+#           only upload the latest built artefact
+# 'once'  : only once upload any built artefact for the build target
+# 'none'  : do not upload any built artefacts
+upload_policy = once
+
+# which GH account has the permission to trigger the deployment (by setting
+# the label 'bot:deploy' (apparently this cannot be restricted on GitHub)
+# if value is left/empty everyone can trigger the deployment
+# value can be a space delimited list of GH accounts
+deploy_permission = boegel trz42 bedroge

From 69f662795dca88559f4885e5fa80a16c0da655fe Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 17 Mar 2023 22:13:21 +0100
Subject: [PATCH 128/157] improve handling of potentially non existing list
 files

---
 create_tarball.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/create_tarball.sh b/create_tarball.sh
index 9381407d10..fd04a195cb 100755
--- a/create_tarball.sh
+++ b/create_tarball.sh
@@ -55,7 +55,7 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then
         | grep -v '/\.wh\.' | sed -e 's/.lua$//' | awk -F'/' '{printf "%s/%s\n", $(NF-1), $NF}' | sort | uniq \
         >> ${module_files_list}
 fi
-if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software ]; then
+if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then
     # installation directories but only those for which module files were created
     for package_version in $(cat ${module_files_list}); do
         echo "handling ${package_version}"
@@ -66,9 +66,9 @@ fi
 
 # add a bit debug output
 echo "wrote file list to ${files_list}"
-cat ${files_list}
+[ -r ${files_list} ] && cat ${files_list}
 echo "wrote module file list to ${module_files_list}"
-cat ${module_files_list}
+[ -r ${module_files_list} ] && cat ${module_files_list}
 
 topdir=${cvmfs_repo}/versions/
 

From a4c8e6471bad5ce80df906c5126f5a02a06c55a0 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Tue, 11 Apr 2023 09:35:44 +0200
Subject: [PATCH 129/157] removing comment to address review comment

---
 check_missing_installations.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/check_missing_installations.sh b/check_missing_installations.sh
index 45f276dee1..926f475903 100755
--- a/check_missing_installations.sh
+++ b/check_missing_installations.sh
@@ -38,9 +38,6 @@ ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experim
 
 grep " required modules missing:" ${eb_missing_out} > /dev/null
 
-# we need to process the result (from finding `No missing modules` to NOT finding
-# ` required modules missing:` and no other error happened)
-#
 # if grep returns 1 (` required modules missing:` was NOT found), we set
 # MODULES_MISSING to 0, otherwise (it was found or another error) we set it to 1
 [[ $? -eq 1 ]] && MODULES_MISSING=0 || MODULES_MISSING=1

From 4d132273569f6d40e503bdd48b5b3aca88f7c907 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Tue, 11 Apr 2023 12:34:39 +0200
Subject: [PATCH 130/157] update determining directories to tar + adding a note
 on assumptions made

---
 create_tarball.sh | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/create_tarball.sh b/create_tarball.sh
index fd04a195cb..b6c72b341d 100755
--- a/create_tarball.sh
+++ b/create_tarball.sh
@@ -52,11 +52,16 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then
     find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list}
     # module files and symlinks
     find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules/all -type f -o -type l \
-        | grep -v '/\.wh\.' | sed -e 's/.lua$//' | awk -F'/' '{printf "%s/%s\n", $(NF-1), $NF}' | sort | uniq \
+        | grep -v '/\.wh\.' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \
         >> ${module_files_list}
 fi
 if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then
     # installation directories but only those for which module files were created
+    # Note, we assume that module names (as defined by 'PACKAGE_NAME/VERSION.lua'
+    # using EasyBuild's standard module naming scheme) match the name of the
+    # software installation directory (expected to be 'PACKAGE_NAME/VERSION/').
+    # If either side changes (module naming scheme or naming of software
+    # installation directories), the procedure will likely not work.
     for package_version in $(cat ${module_files_list}); do
         echo "handling ${package_version}"
         ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \

From 4da2a4d4c779b0f04cdb15a8d574a5e4d0f7f33c Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 24 Apr 2023 12:04:46 +0200
Subject: [PATCH 131/157] check if eb ran successful and redirect stderr

---
 check_missing_installations.sh | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/check_missing_installations.sh b/check_missing_installations.sh
index 926f475903..4a5316c09f 100755
--- a/check_missing_installations.sh
+++ b/check_missing_installations.sh
@@ -3,6 +3,7 @@
 # Script to check for missing installations in EESSI pilot software stack (version 2021.12)
 #
 # author: Kenneth Hoste (@boegel)
+# author: Thomas Roeblitz (@trz42)
 #
 # license: GPLv2
 #
@@ -21,12 +22,15 @@ source $TOPDIR/scripts/utils.sh
 source $TOPDIR/configure_easybuild
 
 echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..."
-ok_msg="No missing installations, party time!"
-fail_msg="On no, some installations are still missing, how did that happen?!"
 eb_missing_out=$LOCAL_TMPDIR/eb_missing.out
 # we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used
 # PR #16531: Nextflow-22.10.1.eb
-${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out}
+${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing 2>&1 | tee ${eb_missing_out}
+exit_code=${PIPESTATUS[0]}
+
+ok_msg="Command 'eb --missing ...' succeeded, analysing output..."
+fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'"
+check_exit_code ${exit_code} "${ok_msg}" "${fail_msg}"
 
 # the above assesses the installed software for each easyconfig provided in
 # the easystack file and then print messages such as
@@ -37,8 +41,11 @@ ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experim
 # output does not contain any line with ` required modules missing:`
 
 grep " required modules missing:" ${eb_missing_out} > /dev/null
+exit_code=$?
 
 # if grep returns 1 (` required modules missing:` was NOT found), we set
 # MODULES_MISSING to 0, otherwise (it was found or another error) we set it to 1
-[[ $? -eq 1 ]] && MODULES_MISSING=0 || MODULES_MISSING=1
+[[ ${exit_code} -eq 1 ]] && MODULES_MISSING=0 || MODULES_MISSING=1
+ok_msg="No missing installations, party time!"
+fail_msg="On no, some installations are still missing, how did that happen?!"
 check_exit_code ${MODULES_MISSING} "${ok_msg}" "${fail_msg}"

From 49d41153d1a4bac24ebb77eaf84420ed42400c23 Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@cecam.org>
Date: Thu, 27 Apr 2023 14:20:01 +0200
Subject: [PATCH 132/157] Make sure installations are read only

This should help us avoid the issue we saw with `.pyc` files
---
 configure_easybuild | 1 +
 1 file changed, 1 insertion(+)

diff --git a/configure_easybuild b/configure_easybuild
index 19b2d7454b..245553f342 100644
--- a/configure_easybuild
+++ b/configure_easybuild
@@ -13,6 +13,7 @@ export EASYBUILD_ZIP_LOGS=bzip2
 
 export EASYBUILD_RPATH=1
 export EASYBUILD_FILTER_ENV_VARS=LD_LIBRARY_PATH
+export EASYBUILD_READ_ONLY_INSTALLDIR=1
 
 # assume that eb_hooks.py is located in same directory as this script (configure_easybuild)
 TOPDIR=$(dirname $(realpath $BASH_SOURCE))

From 8620428abb6fae254e62609536fc1afef6ccb6ed Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 16 May 2023 14:08:25 +0200
Subject: [PATCH 133/157] add replacement init script for removed 2021.06 pilot
 version

---
 versions/2021.06/init/Magic_Castle/bash       |  3 +++
 versions/2021.06/init/bash                    |  3 +++
 .../2021.06/init/print_deprecation_warning.sh | 19 +++++++++++++++++++
 3 files changed, 25 insertions(+)
 create mode 100644 versions/2021.06/init/Magic_Castle/bash
 create mode 100644 versions/2021.06/init/bash
 create mode 100755 versions/2021.06/init/print_deprecation_warning.sh

diff --git a/versions/2021.06/init/Magic_Castle/bash b/versions/2021.06/init/Magic_Castle/bash
new file mode 100644
index 0000000000..5f149c817f
--- /dev/null
+++ b/versions/2021.06/init/Magic_Castle/bash
@@ -0,0 +1,3 @@
+/cvmfs/pilot.eessi-hpc.org/versions/2021.06/init/print_deprecation_warning.sh
+
+source /cvmfs/pilot.eessi-hpc.org/versions/2021.12/init/Magic_Castle/bash
diff --git a/versions/2021.06/init/bash b/versions/2021.06/init/bash
new file mode 100644
index 0000000000..82a078849a
--- /dev/null
+++ b/versions/2021.06/init/bash
@@ -0,0 +1,3 @@
+/cvmfs/pilot.eessi-hpc.org/versions/2021.06/init/print_deprecation_warning.sh
+
+source /cvmfs/pilot.eessi-hpc.org/versions/2021.12/init/bash
diff --git a/versions/2021.06/init/print_deprecation_warning.sh b/versions/2021.06/init/print_deprecation_warning.sh
new file mode 100755
index 0000000000..b721ed2f71
--- /dev/null
+++ b/versions/2021.06/init/print_deprecation_warning.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+function echo_yellow_stderr() {
+    echo -e "\e[33m${1}\e[0m" >&2
+}
+
+echo_yellow_stderr
+echo_yellow_stderr "WARNING: Version 2021.06 of the EESSI pilot repository has been removed since 16 May 2023."
+echo_yellow_stderr
+echo_yellow_stderr "Version 2021.12 of the EESSI pilot repository can be used as a drop-in replacement, "
+echo_yellow_stderr "so we have prepared your environment to use that instead."
+echo_yellow_stderr
+echo_yellow_stderr "In the future, please run"
+echo_yellow_stderr
+echo_yellow_stderr "    source /cvmfs/pilot.eessi-hpc.org/latest/init/bash"
+echo_yellow_stderr
+echo_yellow_stderr "to prepare your start using the EESSI pilot repository."
+echo_yellow_stderr
+echo_yellow_stderr "See also https://eessi.github.io/docs/using_eessi/setting_up_environment ."
+echo_yellow_stderr

From fc3573b8a975cc94c50d89ca4aa066cd17993ca1 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 18 May 2023 09:16:13 +0200
Subject: [PATCH 134/157] don't print warning when unknown targets are found
 (fixes #247)

---
 init/eessi_software_subdir_for_host.py | 2 --
 init/test.py                           | 4 ++--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/init/eessi_software_subdir_for_host.py b/init/eessi_software_subdir_for_host.py
index b09b4711e4..58e9cfd2e6 100755
--- a/init/eessi_software_subdir_for_host.py
+++ b/init/eessi_software_subdir_for_host.py
@@ -101,8 +101,6 @@ def find_best_target(eessi_prefix):
             continue
         if uarch in KNOWN_CPU_UARCHS:
             target_uarchs.append(KNOWN_CPU_UARCHS[uarch])
-        else:
-            warning('Ignoring unknown target "%s"' % uarch)
 
     host_uarch = KNOWN_CPU_UARCHS[host_cpu_name]
     compat_target_uarchs = sorted([x for x in target_uarchs if x <= host_uarch])
diff --git a/init/test.py b/init/test.py
index 0ed09abd6f..f10be5e66e 100644
--- a/init/test.py
+++ b/init/test.py
@@ -60,12 +60,12 @@ def broadwell_host_triple():
     prep_tmpdir(tmpdir, ['x86_64/intel/ivybridge'])
     assert find_best_target(tmpdir) == 'x86_64/intel/ivybridge'
 
-    # unknown targets don't cause trouble (only warning)
+    # unknown targets don't cause trouble
     prep_tmpdir(tmpdir, ['x86_64/intel/no_such_intel_cpu'])
     assert find_best_target(tmpdir) == 'x86_64/intel/ivybridge'
     captured = capsys.readouterr()
     assert captured.out == ''
-    assert captured.err == 'WARNING: Ignoring unknown target "no_such_intel_cpu"\n'
+    assert captured.err == ''
 
     # older targets have to no impact on best target (sandybridge < ivybridge)
     prep_tmpdir(tmpdir, ['x86_64/intel/sandybridge'])

From e688ec8de4b6684296de00f31eba1578cbc0d72c Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 18 May 2023 10:01:24 +0200
Subject: [PATCH 135/157] avoid that versions/2021.06/init is included in init
 tarball

---
 create_directory_tarballs.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/create_directory_tarballs.sh b/create_directory_tarballs.sh
index be282463eb..70e666f871 100755
--- a/create_directory_tarballs.sh
+++ b/create_directory_tarballs.sh
@@ -26,7 +26,7 @@ fi
 tartmp=$(mktemp -t -d init.XXXXX)
 mkdir "${tartmp}/${version}"
 tarname="eessi-${version}-init-$(date +%s).tar.gz"
-curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --strip-components=1 --wildcards */init/
+curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --strip-components=1 --no-wildcards-match-slash --wildcards '*/init/'
 source "${tartmp}/${version}/init/minimal_eessi_env"
 if [ "${EESSI_PILOT_VERSION}" != "${version}" ]
 then
@@ -42,7 +42,7 @@ echo_green "Done! Created tarball ${tarname}."
 tartmp=$(mktemp -t -d scripts.XXXXX)
 mkdir "${tartmp}/${version}"
 tarname="eessi-${version}-scripts-$(date +%s).tar.gz"
-curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --strip-components=1 --wildcards '*/scripts/'
+curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --strip-components=1 --no-wildcards-match-slash --wildcards '*/scripts/'
 tar czf "${tarname}" -C "${tartmp}" "${version}"
 rm -rf "${tartmp}"
 

From 7554604828ab97c6026ccaa6194213fb492329b3 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 27 May 2023 17:27:58 +0200
Subject: [PATCH 136/157] implement post-prepare hook for GCCcore to also have
 a wrapper in place with system type prefix like 'x86_64-pc-linux-gnu'

---
 eb_hooks.py | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/eb_hooks.py b/eb_hooks.py
index df7742f999..9b07ed3b8e 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -3,8 +3,11 @@
 import os
 import re
 
+from easybuild.easyblocks.generic.configuremake import obtain_config_guess
 from easybuild.tools.build_log import EasyBuildError, print_msg
 from easybuild.tools.config import build_option, update_build_option
+from easybuild.tools.filetools import copy_file, which
+from easybuild.tools.run import run_cmd
 from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_cpu_features
 from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC
 
@@ -89,6 +92,25 @@ def pre_prepare_hook(self, *args, **kwargs):
                   mpi_family, rpath_override_dirs)
 
 
+def gcc_postprepare(self, *args, **kwargs):
+    """
+    Post-configure hook for GCCcore:
+    - copy RPATH wrapper script for linker commands to also have a wrapper in place with system type prefix like 'x86_64-pc-linux-gnu'
+    """
+    if self.name == 'GCCcore':
+        config_guess = obtain_config_guess()
+        system_type, _ = run_cmd(config_guess, log_all=True)
+        cmd_prefix = '%s-' % system_type.strip()
+        for cmd in ('ld', 'ld.gold', 'ld.bfd'):
+            wrapper = which(cmd)
+            self.log.info("Path to %s wrapper: %s" % (cmd, wrapper))
+            wrapper_dir = os.path.dirname(wrapper)
+            prefix_wrapper = os.path.join(wrapper_dir, cmd_prefix + cmd)
+            copy_file(wrapper, prefix_wrapper)
+            self.log.info("Path to %s wrapper with '%s' prefix: %s" % (cmd, cmd_prefix, which(prefix_wrapper)))
+    else:
+        raise EasyBuildError("GCCcore-specific hook triggered for non-GCCcore easyconfig?!")
+
 def post_prepare_hook(self, *args, **kwargs):
     """Main post-prepare hook: trigger custom functions."""
 
@@ -98,6 +120,9 @@ def post_prepare_hook(self, *args, **kwargs):
         print_msg("Resetting rpath_override_dirs to original value: %s", getattr(self, EESSI_RPATH_OVERRIDE_ATTR))
         delattr(self, EESSI_RPATH_OVERRIDE_ATTR)
 
+    if self.name in POST_PREPARE_HOOKS:
+        POST_PREPARE_HOOKS[self.name](self, *args, **kwargs)
+
 
 def cgal_toolchainopts_precise(ec, eprefix):
     """Enable 'precise' rather than 'strict' toolchain option for CGAL on POWER."""
@@ -187,6 +212,10 @@ def wrf_preconfigure(self, *args, **kwargs):
     'UCX': ucx_eprefix,
 }
 
+POST_PREPARE_HOOKS = {
+    'GCCcore': gcc_postprepare,
+}
+
 PRE_CONFIGURE_HOOKS = {
     'libfabric': libfabric_disable_psm3_x86_64_generic,
     'MetaBAT': metabat_preconfigure,

From c2ec231652a5fe12c0e90e3d9576d6e36ae7fbfa Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 27 May 2023 19:34:35 +0200
Subject: [PATCH 137/157] patch copied wrapper script to make sure it's
 actually working (CMD needs to be set to 'ld' for 'x86_64-pc-linux-gnu-ld',
 because EasyBuild's rpath_args.py only takes into account 'ld' when
 determining whether or not to use -Wl,-rpath or just -rpath

---
 eb_hooks.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index 9b07ed3b8e..777d583c7b 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -6,7 +6,7 @@
 from easybuild.easyblocks.generic.configuremake import obtain_config_guess
 from easybuild.tools.build_log import EasyBuildError, print_msg
 from easybuild.tools.config import build_option, update_build_option
-from easybuild.tools.filetools import copy_file, which
+from easybuild.tools.filetools import apply_regex_substitutions, copy_file, which
 from easybuild.tools.run import run_cmd
 from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_cpu_features
 from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC
@@ -108,6 +108,17 @@ def gcc_postprepare(self, *args, **kwargs):
             prefix_wrapper = os.path.join(wrapper_dir, cmd_prefix + cmd)
             copy_file(wrapper, prefix_wrapper)
             self.log.info("Path to %s wrapper with '%s' prefix: %s" % (cmd, cmd_prefix, which(prefix_wrapper)))
+
+            # we need to tweak the copied wrapper script, so that:
+            regex_subs = [
+                # - CMD in the script is set to the command name without prefix, because EasyBuild's rpath_args.py
+                #   script that is used by the wrapper script only checks for 'ld', 'ld.gold', etc.
+                #   when checking whether or not to use -Wl
+                ('^CMD=.*', 'CMD=%s' % cmd),
+                # - the path to the correct actual binary is logged and called
+                ('/%s ' % cmd, '/%s ' % (cmd_prefix + cmd)),
+            ]
+            apply_regex_substitutions(prefix_wrapper, regex_subs)
     else:
         raise EasyBuildError("GCCcore-specific hook triggered for non-GCCcore easyconfig?!")
 

From 9774323e7c4830d736f54e2675a7bd52375d5737 Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@cecam.org>
Date: Tue, 30 May 2023 14:43:31 +0200
Subject: [PATCH 138/157] Update rpath injection hook for `versions` subdir

---
 eb_hooks.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index 777d583c7b..2fba925b01 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -27,12 +27,11 @@ def get_eessi_envvar(eessi_envvar):
 def get_rpath_override_dirs(software_name):
     # determine path to installations in software layer via $EESSI_SOFTWARE_PATH
     eessi_software_path = get_eessi_envvar('EESSI_SOFTWARE_PATH')
-    eessi_pilot_version = get_eessi_envvar('EESSI_PILOT_VERSION')
 
     # construct the rpath override directory stub
     rpath_injection_stub = os.path.join(
         # Make sure we are looking inside the `host_injections` directory
-        eessi_software_path.replace(eessi_pilot_version, os.path.join('host_injections', eessi_pilot_version), 1),
+        eessi_software_path.replace('versions', 'host_injections', 1),
         # Add the subdirectory for the specific software
         'rpath_overrides',
         software_name,

From bb37c3197974ca37af13046881084a3cd2b67104 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 30 May 2023 15:17:13 +0200
Subject: [PATCH 139/157] flesh out load_easybuild_module.sh script from
 EESSI-pilot-install-software.sh

---
 .github/workflows/tests_scripts.yml |  50 ++++++++++++-
 EESSI-pilot-install-software.sh     |  66 +----------------
 load_easybuild_module.sh            | 108 ++++++++++++++++++++++++++++
 3 files changed, 158 insertions(+), 66 deletions(-)
 create mode 100755 load_easybuild_module.sh

diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml
index 94eb2894ee..1c82b04872 100644
--- a/.github/workflows/tests_scripts.yml
+++ b/.github/workflows/tests_scripts.yml
@@ -4,22 +4,26 @@ on:
   push:
     paths:
       - build_container.sh
+      - create_directory_tarballs.sh
+      - EESSI-pilot-install-software.sh
       - install_software_layer.sh
+      - load_easybuild_module.sh
       - run_in_compat_layer_env.sh
       - scripts/utils.sh
       - update_lmod_cache.sh
-      - create_directory_tarballs.sh
 
   pull_request:
     branches:
       - main
     paths:
       - build_container.sh
+      - create_directory_tarballs.sh
+      - EESSI-pilot-install-software.sh
       - install_software_layer.sh
+      - load_easybuild_module.sh
       - run_in_compat_layer_env.sh
       - scripts/utils.sh
       - update_lmod_cache.sh
-      - create_directory_tarballs.sh
 permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
@@ -33,6 +37,48 @@ jobs:
       run: |
         ./install_apptainer_ubuntu.sh
 
+    - name: test load_easybuild_module.sh script
+      run: |
+        export TMPDIR=$(mktemp -d)
+
+        # run tests from /tmp where scripts were copied into,
+        # since scripts must be accessible from within build container
+        cp -a * /tmp/
+        cd /tmp
+
+        for EB_VERSION in '4.5.0' '4.5.1' '4.7.2'; do
+          # Create script that uses load_easybuild_module.sh which we can run in compat layer environment
+          # note: Be careful with single vs double quotes below!
+          #       ${TMPDIR} and ${EB_VERSION} should be expanded, so use double quotes;
+          #       For statements using variables that are only defined in the script, like ${EASYBUILD_INSTALLPATH},
+          #       use single quotes to avoid expansion while creating the script.
+          test_script="${TMPDIR}/eb-${EB_VERSION}.sh"
+          echo '#!/bin/bash' > ${test_script}
+          # both $EB and $TMPDIR environment must be set, required by load_easybuild_module.sh script
+          echo 'export EB="eb"' >> ${test_script}
+          echo "export TMPDIR=${TMPDIR}" >> ${test_script}
+          # set up environment to have utility functions in place that load_easybuild_module.sh script relies on,
+          # along with $EESSI_* environment variables, and Lmod
+          echo 'source /tmp/scripts/utils.sh' >> ${test_script}
+          echo 'source /tmp/init/eessi_environment_variables' >> ${test_script}
+          echo 'source ${EPREFIX}/usr/share/Lmod/init/bash' >> ${test_script}
+          # minimal configuration for EasyBuild so we can test installation aspect of load_easybuild_module.sh script
+          echo "export EASYBUILD_INSTALLPATH=${TMPDIR}" >> ${test_script}
+          echo 'module use ${EASYBUILD_INSTALLPATH}/modules/all' >> ${test_script}
+          echo '' >> ${test_script}
+          echo "source /tmp/load_easybuild_module.sh ${EB_VERSION}" >> ${test_script}
+          echo 'module list' >> ${test_script}
+          echo 'eb --version' >> ${test_script}
+
+          chmod u+x ${test_script}
+
+          # run wrapper script + capture & check output
+          out="${TMPDIR}/eb-${EB_VERSION}.out"
+          ./build_container.sh run /tmp/$USER/EESSI /tmp/run_in_compat_layer_env.sh ${test_script} 2>&1 | tee ${out}
+          pattern="^This is EasyBuild ${EB_VERSION} "
+          grep "${pattern}" ${out} || (echo "Pattern '${pattern}' not found in output!" && exit 1)
+        done
+
     - name: test install_software_layer.sh script
       run: |
           # scripts need to be copied to /tmp,
diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
index c05ccf8ab9..d9bcf20231 100755
--- a/EESSI-pilot-install-software.sh
+++ b/EESSI-pilot-install-software.sh
@@ -139,70 +139,8 @@ fi
 
 REQ_EB_VERSION='4.5.0'
 
-echo ">> Checking for EasyBuild module..."
-ml_av_easybuild_out=$TMPDIR/ml_av_easybuild.out
-module avail 2>&1 | grep -i easybuild/${REQ_EB_VERSION} &> ${ml_av_easybuild_out}
-if [[ $? -eq 0 ]]; then
-    echo_green ">> EasyBuild module found!"
-else
-    echo_yellow ">> No EasyBuild module yet, installing it..."
-
-    EB_TMPDIR=${TMPDIR}/ebtmp
-    echo ">> Temporary installation (in ${EB_TMPDIR})..."
-    pip_install_out=${TMPDIR}/pip_install.out
-    pip3 install --prefix $EB_TMPDIR easybuild &> ${pip_install_out}
-
-    # keep track of original $PATH and $PYTHONPATH values, so we can restore them
-    ORIG_PATH=$PATH
-    ORIG_PYTHONPATH=$PYTHONPATH
-
-    echo ">> Final installation in ${EASYBUILD_INSTALLPATH}..."
-    export PATH=${EB_TMPDIR}/bin:$PATH
-    export PYTHONPATH=$(ls -d ${EB_TMPDIR}/lib/python*/site-packages):$PYTHONPATH
-    eb_install_out=${TMPDIR}/eb_install.out
-    ok_msg="Latest EasyBuild release installed, let's go!"
-    fail_msg="Installing latest EasyBuild release failed, that's not good... (output: ${eb_install_out})"
-    eb --install-latest-eb-release &> ${eb_install_out}
-    check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-    eb --search EasyBuild-${REQ_EB_VERSION}.eb | grep EasyBuild-${REQ_EB_VERSION}.eb > /dev/null
-    if [[ $? -eq 0 ]]; then
-        ok_msg="EasyBuild v${REQ_EB_VERSION} installed, alright!"
-        fail_msg="Installing EasyBuild v${REQ_EB_VERSION}, yikes! (output: ${eb_install_out})"
-        eb EasyBuild-${REQ_EB_VERSION}.eb >> ${eb_install_out} 2>&1
-        check_exit_code $? "${ok_msg}" "${fail_msg}"
-    fi
-
-    # restore origin $PATH and $PYTHONPATH values
-    export PATH=${ORIG_PATH}
-    export PYTHONPATH=${ORIG_PYTHONPATH}
-
-    module avail easybuild/${REQ_EB_VERSION} &> ${ml_av_easybuild_out}
-    if [[ $? -eq 0 ]]; then
-        echo_green ">> EasyBuild module installed!"
-    else
-        fatal_error "EasyBuild/${REQ_EB_VERSION} module failed to install?! (output of 'pip install' in ${pip_install_out}, output of 'eb' in ${eb_install_out}, output of 'ml av easybuild' in ${ml_av_easybuild_out})"
-    fi
-fi
-
-echo ">> Loading EasyBuild module..."
-module load EasyBuild/$REQ_EB_VERSION
-eb_show_system_info_out=${TMPDIR}/eb_show_system_info.out
-$EB --show-system-info > ${eb_show_system_info_out}
-if [[ $? -eq 0 ]]; then
-    echo_green ">> EasyBuild seems to be working!"
-    $EB --version | grep "${REQ_EB_VERSION}"
-    if [[ $? -eq 0 ]]; then
-        echo_green "Found EasyBuild version ${REQ_EB_VERSION}, looking good!"
-    else
-        $EB --version
-        fatal_error "Expected to find EasyBuild version ${REQ_EB_VERSION}, giving up here..."
-    fi
-    $EB --show-config
-else
-    cat ${eb_show_system_info_out}
-    fatal_error "EasyBuild not working?!"
-fi
+# load EasyBuild module (will be installed if it's not available yet)
+source ${TOPDIR}/load_easybuild_module.sh ${REQ_EB_VERSION}
 
 echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALLPATH}..."
 
diff --git a/load_easybuild_module.sh b/load_easybuild_module.sh
new file mode 100755
index 0000000000..1f9172fa6f
--- /dev/null
+++ b/load_easybuild_module.sh
@@ -0,0 +1,108 @@
+# Script to load the environment module for a specific version of EasyBuild.
+# If that module is not available yet, the current latest EasyBuild version of EasyBuild will be installed,
+# and used to install the specific EasyBuild version being specified.
+#
+# This script must be sourced, since it makes changes in the current environment, like loading an EasyBuild module.
+
+set -o pipefail
+
+if [ $# -ne 1 ]; then
+    echo "Usage: $0 <EasyBuild version>" >&2
+    exit 1
+fi
+
+# don't use $EB_VERSION, since that enables always running 'eb --version'
+EB_VERSION=${1}
+
+# make sure that environment variables that we expect to be set are indeed set
+if [ -z "${TMPDIR}" ]; then 
+    echo "\$TMPDIR is not set" >&2
+    exit 2
+fi
+
+if [ -z "${EB}" ]; then 
+    echo "\$EB is not set" >&2
+    exit 2
+fi
+
+# make sure that utility functions are defined (cfr. scripts/utils.sh script in EESSI/software-layer repo)
+type check_exit_code
+if [ $? -ne 0 ]; then
+    echo "check_exit_code function is not defined" >&2
+    exit 3
+fi
+
+echo ">> Checking for EasyBuild module..."
+
+ml_av_easybuild_out=${TMPDIR}/ml_av_easybuild.out
+module avail 2>&1 | grep -i easybuild/${EB_VERSION} &> ${ml_av_easybuild_out}
+
+if [[ $? -eq 0 ]]; then
+    echo_green ">> Module for EasyBuild v${EB_VERSION} found!"
+else
+    echo_yellow ">> No module yet for EasyBuild v${EB_VERSION}, installing it..."
+
+    EB_TMPDIR=${TMPDIR}/ebtmp
+    echo ">> Temporary installation (in ${EB_TMPDIR})..."
+    pip_install_out=${TMPDIR}/pip_install.out
+    pip3 install --prefix ${EB_TMPDIR} easybuild &> ${pip_install_out}
+
+    # keep track of original $PATH and $PYTHONPATH values, so we can restore them
+    ORIG_PATH=${PATH}
+    ORIG_PYTHONPATH=${PYTHONPATH}
+
+    echo ">> Final installation in ${EASYBUILD_INSTALLPATH}..."
+    export PATH=${EB_TMPDIR}/bin:${PATH}
+    export PYTHONPATH=$(ls -d ${EB_TMPDIR}/lib/python*/site-packages):${PYTHONPATH}
+    eb_install_out=${TMPDIR}/eb_install.out
+    ok_msg="Latest EasyBuild release installed, let's go!"
+    fail_msg="Installing latest EasyBuild release failed, that's not good... (output: ${eb_install_out})"
+    ${EB} --install-latest-eb-release 2>&1 | tee ${eb_install_out}
+    check_exit_code $? "${ok_msg}" "${fail_msg}"
+
+    eb_ec=EasyBuild-${EB_VERSION}.eb
+    ${EB} --search ${eb_ec} | grep ${eb_ec} > /dev/null
+    if [[ $? -eq 0 ]]; then
+        echo "Easyconfig found for EasyBuild v${EB_VERSION}, so installing it..."
+        ok_msg="EasyBuild v${EB_VERSION} installed, alright!"
+        fail_msg="Installing EasyBuild v${EB_VERSION}, yikes! (output: ${eb_install_out})"
+        #${EB} EasyBuild-${EB_VERSION}.eb >> ${eb_install_out} 2>&1
+        ${EB} EasyBuild-${EB_VERSION}.eb 2>&1 | tee -a ${eb_install_out}
+        check_exit_code $? "${ok_msg}" "${fail_msg}"
+    else
+        echo "No easyconfig found for EasyBuild v${EB_VERSION}"
+    fi
+
+    # restore origin $PATH and $PYTHONPATH values, and clean up environment variables that are no longer needed
+    export PATH=${ORIG_PATH}
+    export PYTHONPATH=${ORIG_PYTHONPATH}
+    unset EB_TMPDIR ORIG_PATH ORIG_PYTHONPATH
+
+    module avail easybuild/${EB_VERSION} &> ${ml_av_easybuild_out}
+    if [[ $? -eq 0 ]]; then
+        echo_green ">> EasyBuild module installed!"
+    else
+        fatal_error "EasyBuild/${EB_VERSION} module failed to install?! (output of 'pip install' in ${pip_install_out}, output of 'eb' in ${eb_install_out}, output of 'ml av easybuild' in ${ml_av_easybuild_out})"
+    fi
+fi
+
+echo ">> Loading EasyBuild v${EB_VERSION} module..."
+module load EasyBuild/${EB_VERSION}
+eb_show_system_info_out=${TMPDIR}/eb_show_system_info.out
+${EB} --show-system-info > ${eb_show_system_info_out}
+if [[ $? -eq 0 ]]; then
+    echo_green ">> EasyBuild seems to be working!"
+    ${EB} --version | grep "${EB_VERSION}"
+    if [[ $? -eq 0 ]]; then
+        echo_green "Found EasyBuild version ${EB_VERSION}, looking good!"
+    else
+        ${EB} --version
+        fatal_error "Expected to find EasyBuild version ${EB_VERSION}, giving up here..."
+    fi
+    ${EB} --show-config
+else
+    cat ${eb_show_system_info_out}
+    fatal_error "EasyBuild not working?!"
+fi
+
+unset EB_VERSION

From abb931cfb308f03c3cb36846a9d98ee217a6b1b4 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sun, 4 Jun 2023 13:54:35 +0200
Subject: [PATCH 140/157] use eessi_container.sh rather than build_container.sh
 script in test workflow for load_easybuild_module.sh script

---
 .github/workflows/tests_scripts.yml | 27 ++++++++++++---------------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml
index 1c82b04872..5c0b3893ae 100644
--- a/.github/workflows/tests_scripts.yml
+++ b/.github/workflows/tests_scripts.yml
@@ -39,42 +39,39 @@ jobs:
 
     - name: test load_easybuild_module.sh script
       run: |
-        export TMPDIR=$(mktemp -d)
-
-        # run tests from /tmp where scripts were copied into,
-        # since scripts must be accessible from within build container
-        cp -a * /tmp/
-        cd /tmp
+        # bind current directory into container as /software-layer
+        export SINGULARITY_BIND="${PWD}:/software-layer"
 
         for EB_VERSION in '4.5.0' '4.5.1' '4.7.2'; do
           # Create script that uses load_easybuild_module.sh which we can run in compat layer environment
           # note: Be careful with single vs double quotes below!
-          #       ${TMPDIR} and ${EB_VERSION} should be expanded, so use double quotes;
+          #       ${EB_VERSION} should be expanded, so use double quotes;
           #       For statements using variables that are only defined in the script, like ${EASYBUILD_INSTALLPATH},
           #       use single quotes to avoid expansion while creating the script.
-          test_script="${TMPDIR}/eb-${EB_VERSION}.sh"
+          test_script="${PWD}/eb-${EB_VERSION}.sh"
           echo '#!/bin/bash' > ${test_script}
           # both $EB and $TMPDIR environment must be set, required by load_easybuild_module.sh script
           echo 'export EB="eb"' >> ${test_script}
-          echo "export TMPDIR=${TMPDIR}" >> ${test_script}
+          echo 'export TMPDIR=$(mktemp -d)' >> ${test_script}
           # set up environment to have utility functions in place that load_easybuild_module.sh script relies on,
           # along with $EESSI_* environment variables, and Lmod
-          echo 'source /tmp/scripts/utils.sh' >> ${test_script}
-          echo 'source /tmp/init/eessi_environment_variables' >> ${test_script}
+          echo 'ls -l /software-layer/' >> ${test_script}
+          echo 'source /software-layer/scripts/utils.sh' >> ${test_script}
+          echo 'source /software-layer/init/eessi_environment_variables' >> ${test_script}
           echo 'source ${EPREFIX}/usr/share/Lmod/init/bash' >> ${test_script}
           # minimal configuration for EasyBuild so we can test installation aspect of load_easybuild_module.sh script
-          echo "export EASYBUILD_INSTALLPATH=${TMPDIR}" >> ${test_script}
+          echo "export EASYBUILD_INSTALLPATH=/tmp/eb-${EB_VERSION}" >> ${test_script}
           echo 'module use ${EASYBUILD_INSTALLPATH}/modules/all' >> ${test_script}
           echo '' >> ${test_script}
-          echo "source /tmp/load_easybuild_module.sh ${EB_VERSION}" >> ${test_script}
+          echo "source /software-layer/load_easybuild_module.sh ${EB_VERSION}" >> ${test_script}
           echo 'module list' >> ${test_script}
           echo 'eb --version' >> ${test_script}
 
           chmod u+x ${test_script}
 
           # run wrapper script + capture & check output
-          out="${TMPDIR}/eb-${EB_VERSION}.out"
-          ./build_container.sh run /tmp/$USER/EESSI /tmp/run_in_compat_layer_env.sh ${test_script} 2>&1 | tee ${out}
+          out="${PWD}/eb-${EB_VERSION}.out"
+          ./eessi_container.sh --access rw --mode run --verbose /software-layer/run_in_compat_layer_env.sh /software-layer/eb-${EB_VERSION}.sh 2>&1 | tee ${out}
           pattern="^This is EasyBuild ${EB_VERSION} "
           grep "${pattern}" ${out} || (echo "Pattern '${pattern}' not found in output!" && exit 1)
         done

From fe855f34ab018df6933bedb4632cdf8b1d6a30d6 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sun, 4 Jun 2023 17:31:05 +0200
Subject: [PATCH 141/157] fix remarks for load_easybuild_module.sh script

---
 load_easybuild_module.sh | 32 ++++++++++++++++++++------------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/load_easybuild_module.sh b/load_easybuild_module.sh
index 1f9172fa6f..53b2a4bba2 100755
--- a/load_easybuild_module.sh
+++ b/load_easybuild_module.sh
@@ -11,7 +11,7 @@ if [ $# -ne 1 ]; then
     exit 1
 fi
 
-# don't use $EB_VERSION, since that enables always running 'eb --version'
+# don't use $EASYBUILD_VERSION, since that enables always running 'eb --version'
 EB_VERSION=${1}
 
 # make sure that environment variables that we expect to be set are indeed set
@@ -20,6 +20,8 @@ if [ -z "${TMPDIR}" ]; then
     exit 2
 fi
 
+# ${EB} is used to specify which 'eb' command should be used;
+# can potentially be more than just 'eb', for example when using 'eb --optarch=GENERIC'
 if [ -z "${EB}" ]; then 
     echo "\$EB is not set" >&2
     exit 2
@@ -60,17 +62,23 @@ else
     ${EB} --install-latest-eb-release 2>&1 | tee ${eb_install_out}
     check_exit_code $? "${ok_msg}" "${fail_msg}"
 
-    eb_ec=EasyBuild-${EB_VERSION}.eb
-    ${EB} --search ${eb_ec} | grep ${eb_ec} > /dev/null
+    # maybe the module obtained with --install-latest-eb-release is exactly the EasyBuild version we wanted?
+    module avail 2>&1 | grep -i easybuild/${EB_VERSION} &> ${ml_av_easybuild_out}
     if [[ $? -eq 0 ]]; then
-        echo "Easyconfig found for EasyBuild v${EB_VERSION}, so installing it..."
-        ok_msg="EasyBuild v${EB_VERSION} installed, alright!"
-        fail_msg="Installing EasyBuild v${EB_VERSION}, yikes! (output: ${eb_install_out})"
-        #${EB} EasyBuild-${EB_VERSION}.eb >> ${eb_install_out} 2>&1
-        ${EB} EasyBuild-${EB_VERSION}.eb 2>&1 | tee -a ${eb_install_out}
-        check_exit_code $? "${ok_msg}" "${fail_msg}"
+        echo_green ">> Module for EasyBuild v${EB_VERSION} found!"
     else
-        echo "No easyconfig found for EasyBuild v${EB_VERSION}"
+        eb_ec=EasyBuild-${EB_VERSION}.eb
+        echo_yellow ">> Still no module for EasyBuild v${EB_VERSION}, trying with easyconfig ${eb_ec}..."
+        ${EB} --search ${eb_ec} | grep ${eb_ec} > /dev/null
+        if [[ $? -eq 0 ]]; then
+            echo "Easyconfig ${eb_ec} found for EasyBuild v${EB_VERSION}, so installing it..."
+            ok_msg="EasyBuild v${EB_VERSION} installed, alright!"
+            fail_msg="Installing EasyBuild v${EB_VERSION}, yikes! (output: ${eb_install_out})"
+            ${EB} EasyBuild-${EB_VERSION}.eb 2>&1 | tee -a ${eb_install_out}
+            check_exit_code $? "${ok_msg}" "${fail_msg}"
+        else
+            fatal_error "No easyconfig found for EasyBuild v${EB_VERSION}"
+        fi
     fi
 
     # restore origin $PATH and $PYTHONPATH values, and clean up environment variables that are no longer needed
@@ -80,9 +88,9 @@ else
 
     module avail easybuild/${EB_VERSION} &> ${ml_av_easybuild_out}
     if [[ $? -eq 0 ]]; then
-        echo_green ">> EasyBuild module installed!"
+        echo_green ">> EasyBuild/${EB_VERSION} module installed!"
     else
-        fatal_error "EasyBuild/${EB_VERSION} module failed to install?! (output of 'pip install' in ${pip_install_out}, output of 'eb' in ${eb_install_out}, output of 'ml av easybuild' in ${ml_av_easybuild_out})"
+        fatal_error "EasyBuild/${EB_VERSION} module failed to install?! (output of 'pip install' in ${pip_install_out}, output of 'eb' in ${eb_install_out}, output of 'module avail easybuild' in ${ml_av_easybuild_out})"
     fi
 fi
 

From e687f52d9a1326aad5930c5493462ac26bdf98e8 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Mon, 5 Jun 2023 09:35:18 +0200
Subject: [PATCH 142/157] add author + license to load_easybuild_module.sh

---
 load_easybuild_module.sh | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/load_easybuild_module.sh b/load_easybuild_module.sh
index 53b2a4bba2..4ff2a3c37c 100755
--- a/load_easybuild_module.sh
+++ b/load_easybuild_module.sh
@@ -3,7 +3,15 @@
 # and used to install the specific EasyBuild version being specified.
 #
 # This script must be sourced, since it makes changes in the current environment, like loading an EasyBuild module.
-
+#
+# This script is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Kenneth Hosye (@boegel, HPC-UGent)
+#
+# license: GPLv2
+#
+#
 set -o pipefail
 
 if [ $# -ne 1 ]; then

From 3fc17467b9665724eeae6651c860fd77d28283d0 Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@cecam.org>
Date: Fri, 16 Jun 2023 14:32:20 +0200
Subject: [PATCH 143/157] Allow `archdetect` to print all possible cpu paths

Allow fix incorrect generic path
---
 init/eessi_archdetect.sh | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/init/eessi_archdetect.sh b/init/eessi_archdetect.sh
index d2b6dacf04..97b45bf936 100755
--- a/init/eessi_archdetect.sh
+++ b/init/eessi_archdetect.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-VERSION="1.0.0"
+VERSION="1.1.0"
 
 # Logging
 LOG_LEVEL="INFO"
@@ -105,7 +105,8 @@ cpupath(){
     log "DEBUG" "cpupath: CPU flags of host system: '$cpu_flags'"
   
     # Default to generic CPU
-    local best_arch_match="generic"
+    local best_arch_match="$machine_type/generic"
+    local all_arch_matches=$best_arch_match
   
     # Iterate over the supported CPU specifications to find the best match for host CPU
     # Order of the specifications matters, the last one to match will be selected
@@ -114,22 +115,28 @@ cpupath(){
         if [ "${cpu_vendor}x" == "${arch_spec[1]}x" ]; then
             # each flag in this CPU specification must be found in the list of flags of the host
             check_allinfirst "${cpu_flags[*]}" ${arch_spec[2]} && best_arch_match=${arch_spec[0]} && \
-                log "DEBUG" "cpupath: host CPU best match updated to $best_arch_match"
+                all_arch_matches="$best_arch_match:$all_arch_matches" && \
+                log "DEBUG" "cpupath: host CPU best match updated to $best_arch_match" 
         fi
     done
   
     log "INFO" "cpupath: best match for host CPU: $best_arch_match"
-    echo "$best_arch_match"
+    if [ "allx" == "${1}x" ]; then
+        echo "$all_arch_matches"
+    else
+        echo "$best_arch_match"
+    fi
 }
 
 # Parse command line arguments
-USAGE="Usage: eessi_archdetect.sh [-h][-d] <action>"
+USAGE="Usage: eessi_archdetect.sh [-h][-d][-a] <action>"
 
-while getopts 'hdv' OPTION; do
+while getopts 'hdva' OPTION; do
     case "$OPTION" in
         h) echo "$USAGE"; exit 0;;
         d) LOG_LEVEL="DEBUG";;
         v) echo "eessi_archdetect.sh v$VERSION"; exit 0;;
+        a) all="all";;
         ?) echo "$USAGE"; exit 1;;
     esac
 done
@@ -138,6 +145,6 @@ shift "$(($OPTIND -1))"
 ARGUMENT=${1:-none}
 
 case "$ARGUMENT" in
-    "cpupath") cpupath; exit;;
-    *) echo "$USAGE"; log "ERROR" "Missing <action> argument";;
+    "cpupath") cpupath $all; exit;;
+    *) echo "$USAGE"; log "ERROR" "Missing <action> argument (possible actions: 'cpupath')";;
 esac

From 551d0082bcb15fd0cb90cf958eb4280b03dc2a58 Mon Sep 17 00:00:00 2001
From: Alan O'Cais <alan.ocais@cecam.org>
Date: Fri, 16 Jun 2023 15:44:08 +0200
Subject: [PATCH 144/157] Add tests

---
 .github/workflows/tests_archdetect.yml                     | 7 +++++++
 .../arm/neoverse-n1/AWS-awslinux-graviton2.all.output      | 1 +
 .../arm/neoverse-n1/Azure-Ubuntu20-Altra.all.output        | 1 +
 .../arm/neoverse-v1/AWS-awslinux-graviton3.all.output      | 1 +
 .../ppc64le/power9le/unknown-power9le.all.output           | 1 +
 .../x86_64/amd/zen2/Azure-CentOS7-7V12.all.output          | 1 +
 .../x86_64/amd/zen3/Azure-CentOS7-7V73X.all.output         | 1 +
 .../intel/haswell/archspec-linux-E5-2680-v3.all.output     | 1 +
 .../intel/skylake_avx512/archspec-linux-6132.all.output    | 1 +
 9 files changed, 15 insertions(+)
 create mode 100644 tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.all.output
 create mode 100644 tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.all.output
 create mode 100644 tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.all.output
 create mode 100644 tests/archdetect/ppc64le/power9le/unknown-power9le.all.output
 create mode 100644 tests/archdetect/x86_64/amd/zen2/Azure-CentOS7-7V12.all.output
 create mode 100644 tests/archdetect/x86_64/amd/zen3/Azure-CentOS7-7V73X.all.output
 create mode 100644 tests/archdetect/x86_64/intel/haswell/archspec-linux-E5-2680-v3.all.output
 create mode 100644 tests/archdetect/x86_64/intel/skylake_avx512/archspec-linux-6132.all.output

diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml
index 618f6eb142..cabbaedb6d 100644
--- a/.github/workflows/tests_archdetect.yml
+++ b/.github/workflows/tests_archdetect.yml
@@ -34,3 +34,10 @@ jobs:
               echo "Test for ${{matrix.proc_cpuinfo}} FAILED: $CPU_ARCH" >&2
               exit 1
           fi
+          CPU_ARCHES=$(./init/eessi_archdetect.sh -a cpupath)
+          if [[ $CPU_ARCHES == "$( cat ./tests/archdetect/${{matrix.proc_cpuinfo}}.all.output )" ]]; then
+              echo "Test for ${{matrix.proc_cpuinfo}} PASSED: $CPU_ARCHES" >&2
+          else
+              echo "Test for ${{matrix.proc_cpuinfo}} FAILED: $CPU_ARCHES" >&2
+              exit 1
+          fi
diff --git a/tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.all.output b/tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.all.output
new file mode 100644
index 0000000000..4e4f20c1de
--- /dev/null
+++ b/tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.all.output
@@ -0,0 +1 @@
+aarch64/arm/neoverse-n1:aarch64/generic
\ No newline at end of file
diff --git a/tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.all.output b/tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.all.output
new file mode 100644
index 0000000000..4e4f20c1de
--- /dev/null
+++ b/tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.all.output
@@ -0,0 +1 @@
+aarch64/arm/neoverse-n1:aarch64/generic
\ No newline at end of file
diff --git a/tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.all.output b/tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.all.output
new file mode 100644
index 0000000000..61f4d892f0
--- /dev/null
+++ b/tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.all.output
@@ -0,0 +1 @@
+aarch64/arm/neoverse-v1:aarch64/arm/neoverse-n1:aarch64/generic
\ No newline at end of file
diff --git a/tests/archdetect/ppc64le/power9le/unknown-power9le.all.output b/tests/archdetect/ppc64le/power9le/unknown-power9le.all.output
new file mode 100644
index 0000000000..7ecf79d0a7
--- /dev/null
+++ b/tests/archdetect/ppc64le/power9le/unknown-power9le.all.output
@@ -0,0 +1 @@
+ppc64le/power9le:ppc64le/generic
\ No newline at end of file
diff --git a/tests/archdetect/x86_64/amd/zen2/Azure-CentOS7-7V12.all.output b/tests/archdetect/x86_64/amd/zen2/Azure-CentOS7-7V12.all.output
new file mode 100644
index 0000000000..180de26f0e
--- /dev/null
+++ b/tests/archdetect/x86_64/amd/zen2/Azure-CentOS7-7V12.all.output
@@ -0,0 +1 @@
+x86_64/amd/zen2:x86_64/generic
\ No newline at end of file
diff --git a/tests/archdetect/x86_64/amd/zen3/Azure-CentOS7-7V73X.all.output b/tests/archdetect/x86_64/amd/zen3/Azure-CentOS7-7V73X.all.output
new file mode 100644
index 0000000000..798a0aa565
--- /dev/null
+++ b/tests/archdetect/x86_64/amd/zen3/Azure-CentOS7-7V73X.all.output
@@ -0,0 +1 @@
+x86_64/amd/zen3:x86_64/amd/zen2:x86_64/generic
\ No newline at end of file
diff --git a/tests/archdetect/x86_64/intel/haswell/archspec-linux-E5-2680-v3.all.output b/tests/archdetect/x86_64/intel/haswell/archspec-linux-E5-2680-v3.all.output
new file mode 100644
index 0000000000..a047dd42cc
--- /dev/null
+++ b/tests/archdetect/x86_64/intel/haswell/archspec-linux-E5-2680-v3.all.output
@@ -0,0 +1 @@
+x86_64/intel/haswell:x86_64/generic
\ No newline at end of file
diff --git a/tests/archdetect/x86_64/intel/skylake_avx512/archspec-linux-6132.all.output b/tests/archdetect/x86_64/intel/skylake_avx512/archspec-linux-6132.all.output
new file mode 100644
index 0000000000..c9fa524ea6
--- /dev/null
+++ b/tests/archdetect/x86_64/intel/skylake_avx512/archspec-linux-6132.all.output
@@ -0,0 +1 @@
+x86_64/intel/skylake_avx512:x86_64/intel/haswell:x86_64/generic
\ No newline at end of file

From 919fdca842b2031832f64b8836d43d55ae8b4447 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 23 Jun 2023 13:30:58 +0200
Subject: [PATCH 145/157] rename bot/check-result.sh to bot/check-build.sh

---
 bot/check-build.sh  | 492 ++++++++++++++++++++++++++++++++++++++++++++
 bot/check-result.sh |   1 +
 2 files changed, 493 insertions(+)
 create mode 100755 bot/check-build.sh
 create mode 120000 bot/check-result.sh

diff --git a/bot/check-build.sh b/bot/check-build.sh
new file mode 100755
index 0000000000..ec1ca56bba
--- /dev/null
+++ b/bot/check-build.sh
@@ -0,0 +1,492 @@
+#!/bin/bash
+#
+# Script to check the result of building the EESSI software layer.
+# Intended use is that it is called by a (batch) job running on a compute
+# node.
+#
+# This script is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Thomas Roeblitz (@trz42)
+#
+# license: GPLv2
+#
+
+# result cases
+
+#  - SUCCESS (all of)
+#    - working directory contains slurm-JOBID.out file
+#    - working directory contains eessi*tar.gz
+#    - no message ERROR
+#    - no message FAILED
+#    - no message ' required modules missing:'
+#    - one or more of 'No missing installations'
+#    - message regarding created tarball
+#  - FAILED (one of ... implemented as NOT SUCCESS)
+#    - no slurm-JOBID.out file
+#    - no tarball
+#    - message with ERROR
+#    - message with FAILED
+#    - message with ' required modules missing:'
+#    - no message regarding created tarball
+
+# stop as soon as something fails
+# set -e
+
+TOPDIR=$(dirname $(realpath $0))
+
+source ${TOPDIR}/../scripts/utils.sh
+source ${TOPDIR}/../scripts/cfg_files.sh
+
+# defaults
+export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
+
+# check if ${JOB_CFG_FILE} exists
+if [[ ! -r "${JOB_CFG_FILE}" ]]; then
+    echo_red "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable"
+else
+    echo "bot/check-build.sh: showing ${JOB_CFG_FILE} from software-layer side"
+    cat ${JOB_CFG_FILE}
+
+    echo "bot/check-build.sh: obtaining configuration settings from '${JOB_CFG_FILE}'"
+    cfg_load ${JOB_CFG_FILE}
+fi
+
+display_help() {
+  echo "usage: $0 [OPTIONS]"
+  echo " OPTIONS:"
+  echo "  -h | --help    - display this usage information [default: false]"
+  echo "  -v | --verbose - display more information [default: false]"
+}
+
+# set defaults for command line arguments
+VERBOSE=0
+
+POSITIONAL_ARGS=()
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -h|--help)
+      display_help
+      exit 0
+      ;;
+    -v|--verbose)
+      VERBOSE=1
+      shift 1
+      ;;
+    --)
+      shift
+      POSITIONAL_ARGS+=("$@") # save positional args
+      break
+      ;;
+    -*|--*)
+      fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}"
+      ;;
+    *)  # No more options
+      POSITIONAL_ARGS+=("$1") # save positional arg
+      shift
+      ;;
+  esac
+done
+
+set -- "${POSITIONAL_ARGS[@]}"
+
+job_dir=${PWD}
+
+[[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}"
+
+job_out="slurm-${SLURM_JOB_ID}.out"
+[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'"
+if  [[ -f ${job_out} ]]; then
+    SLURM=1
+    [[ ${VERBOSE} -ne 0 ]] && echo "   found slurm output file '"${job_out}"'"
+else
+    SLURM=0
+    [[ ${VERBOSE} -ne 0 ]] && echo "   Slurm output file '"${job_out}"' NOT found"
+fi
+
+ERROR=-1
+if [[ ${SLURM} -eq 1 ]]; then
+  GP_error='ERROR: '
+  grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_error}")
+  [[ $? -eq 0 ]] && ERROR=1 || ERROR=0
+  # have to be careful to not add searched for pattern into slurm out file
+  [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+fi
+
+FAILED=-1
+if [[ ${SLURM} -eq 1 ]]; then
+  GP_failed='FAILED: '
+  grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
+  [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
+  # have to be careful to not add searched for pattern into slurm out file
+  [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+fi
+
+MISSING=-1
+if [[ ${SLURM} -eq 1 ]]; then
+  GP_req_missing=' required modules missing:'
+  grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_req_missing}")
+  [[ $? -eq 0 ]] && MISSING=1 || MISSING=0
+  # have to be careful to not add searched for pattern into slurm out file
+  [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+fi
+
+NO_MISSING=-1
+if [[ ${SLURM} -eq 1 ]]; then
+  GP_no_missing='No missing installations'
+  grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_no_missing}")
+  [[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0
+  # have to be careful to not add searched for pattern into slurm out file
+  [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+fi
+
+TGZ=-1
+TARBALL=
+if [[ ${SLURM} -eq 1 ]]; then
+  GP_tgz_created="\.tar\.gz created!"
+  grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u)
+  if [[ $? -eq 0 ]]; then
+      TGZ=1
+      TARBALL=$(echo ${grep_out} | sed -e 's@^.*/\(eessi[^/ ]*\) .*$@\1@')
+  else
+      TGZ=0
+  fi
+  # have to be careful to not add searched for pattern into slurm out file
+  [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+fi
+
+[[ ${VERBOSE} -ne 0 ]] && echo "SUMMARY: ${job_dir}/${job_out}"
+[[ ${VERBOSE} -ne 0 ]] && echo "  <test name>: <actual result> (<expected result>)"
+[[ ${VERBOSE} -ne 0 ]] && echo "  ERROR......: $([[ $ERROR -eq 1 ]] && echo 'yes' || echo 'no') (no)"
+[[ ${VERBOSE} -ne 0 ]] && echo "  FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)"
+[[ ${VERBOSE} -ne 0 ]] && echo "  REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)"
+[[ ${VERBOSE} -ne 0 ]] && echo "  NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
+[[ ${VERBOSE} -ne 0 ]] && echo "  TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
+
+job_result_file=_bot_job${SLURM_JOB_ID}.result
+
+if [[ ${SLURM} -eq 1 ]] && \
+   [[ ${ERROR} -eq 0 ]] && \
+   [[ ${FAILED} -eq 0 ]] && \
+   [[ ${MISSING} -eq 0 ]] && \
+   [[ ${NO_MISSING} -eq 1 ]] && \
+   [[ ${TGZ} -eq 1 ]] && \
+   [[ ! -z ${TARBALL} ]]; then
+    # SUCCESS
+    status="SUCCESS"
+    summary=":grin: SUCCESS"
+else
+    # FAILURE
+    status="FAILURE"
+    summary=":cry: FAILURE"
+fi
+
+### Example details/descriptions
+# Note, final string must not contain any line breaks. Below example include
+# line breaks for the sake of readability. In case of FAILURE, the structure is
+# very similar (incl. information about Artefacts if any was produced), however,
+# under Details some lines will be marked with :x:
+# <details>
+#   <summary>:grin: SUCCESS _(click triangle for details)_</summary>
+#   <dl>
+#     <dt>_Details_</dt>
+#     <dd>
+#       :white_check_mark: job output file <code>slurm-4682.out</code><br/>
+#       :white_check_mark: no message matching <code>ERROR: </code><br/>
+#       :white_check_mark: no message matching <code>FAILED: </code><br/>
+#       :white_check_mark: no message matching <code> required modules missing:</code><br/>
+#       :white_check_mark: found message(s) matching <code>No missing installations</code><br/>
+#       :white_check_mark: found message matching <code>tar.gz created!</code><br/>
+#     </dd>
+#     <dt>_Artefacts_</dt>
+#     <dd>
+#       <details>
+#         <summary><code>eessi-2023.06-software-linux-x86_64-generic-1682696567.tar.gz</code></summary>
+#         size: 234 MiB (245366784 bytes)<br/>
+#         entries: 1234<br/>
+#         modules under _2023.06/software/linux/x86_64/generic/modules/all/_<br/>
+#         <pre>
+#           GCC/9.3.0.lua<br/>
+#           GCC/10.3.0.lua<br/>
+#           OpenSSL/1.1.lua
+#         </pre>
+#         software under _2023.06/software/linux/x86_64/generic/software/_
+#         <pre>
+#           GCC/9.3.0/<br/>
+#           CMake/3.20.1-GCCcore-10.3.0/<br/>
+#           OpenMPI/4.1.1-GCC-10.3.0/
+#         </pre>
+#         other under _2023.06/software/linux/x86_64/generic/_
+#         <pre>
+#           .lmod/cache/spiderT.lua<br/>
+#           .lmod/cache/spiderT.luac_5.1<br/>
+#           .lmod/cache/timestamp
+#         </pre>
+#       </details>
+#     </dd>
+#   </dl>
+# </details>
+#
+# <details>
+#   <summary>:cry: FAILURE _(click triangle for details)_</summary>
+#   <dl>
+#     <dt>_Details_</dt>
+#     <dd>
+#       :white_check_mark: job output file <code>slurm-4682.out</code><br/>
+#       :x: no message matching <code>ERROR: </code><br/>
+#       :white_check_mark: no message matching <code>FAILED: </code><br/>
+#       :x: no message matching <code> required modules missing:</code><br/>
+#       :white_check_mark: found message(s) matching <code>No missing installations</code><br/>
+#       :white_check_mark: found message matching <code>tar.gz created!</code><br/>
+#     </dd>
+#     <dt>_Artefacts_</dt>
+#     <dd>
+#       No artefacts were created or found.
+#     </dd>
+#   </dl>
+# </details>
+###
+
+# construct and write complete PR comment details: implements third alternative
+comment_template="<details>__SUMMARY_FMT__<dl>__DETAILS_FMT____ARTEFACTS_FMT__</dl></details>"
+comment_summary_fmt="<summary>__SUMMARY__ _(click triangle for details)_</summary>"
+comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
+comment_success_item_fmt=":white_check_mark: __ITEM__"
+comment_failure_item_fmt=":x: __ITEM__"
+comment_artefacts_fmt="<dt>_Artefacts_</dt><dd>__ARTEFACTS_LIST__</dd>"
+comment_artefact_details_fmt="<details>__ARTEFACT_SUMMARY____ARTEFACT_DETAILS__</details>"
+
+function print_br_item() {
+    format="${1}"
+    item="${2}"
+    echo -n "${format//__ITEM__/${item}}<br/>"
+}
+
+function print_br_item2() {
+    format="${1}"
+    item="${2}"
+    item2="${3}"
+    format1="${format//__ITEM__/${item}}"
+    echo -n "${format1//__ITEM2__/${item2}}<br/>"
+}
+
+function print_code_item() {
+    format="${1}"
+    item="${2}"
+    echo -n "<code>${format//__ITEM__/${item}}</code>"
+}
+
+function print_dd_item() {
+    format="${1}"
+    item="${2}"
+    echo -n "<dd>${format//__ITEM__/${item}}</dd>"
+}
+
+function print_list_item() {
+    format="${1}"
+    item="${2}"
+    echo -n "<li>${format//__ITEM__/${item}}</li>"
+}
+
+function print_pre_item() {
+    format="${1}"
+    item="${2}"
+    echo -n "<pre>${format//__ITEM__/${item}}</pre>"
+}
+
+function success() {
+    format="${comment_success_item_fmt}"
+    item="$1"
+    print_br_item "${format}" "${item}"
+}
+
+function failure() {
+    format="${comment_failure_item_fmt}"
+    item="$1"
+    print_br_item "${format}" "${item}"
+}
+
+function add_detail() {
+    actual=${1}
+    expected=${2}
+    success_msg="${3}"
+    failure_msg="${4}"
+    if [[ ${actual} -eq ${expected} ]]; then
+        success "${success_msg}"
+    else
+        failure "${failure_msg}"
+    fi
+}
+
+echo "[RESULT]" > ${job_result_file}
+echo -n "comment_description = " >> ${job_result_file}
+
+# construct values for placeholders in comment_template:
+# - __SUMMARY_FMT__ -> variable $comment_summary
+# - __DETAILS_FMT__ -> variable $comment_details
+# - __ARTEFACTS_FMT__ -> variable $comment_artefacts
+
+comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
+
+# first construct comment_details_list, abbreviated CoDeList
+# then use it to set comment_details
+CoDeList=""
+
+success_msg="job output file <code>${job_out}</code>"
+failure_msg="no job output file <code>${job_out}</code>"
+CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}")
+
+success_msg="no message matching <code>${GP_error}</code>"
+failure_msg="found message matching <code>${GP_error}</code>"
+CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
+
+success_msg="no message matching <code>${GP_failed}</code>"
+failure_msg="found message matching <code>${GP_failed}</code>"
+CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
+
+success_msg="no message matching <code>${GP_req_missing}</code>"
+failure_msg="found message matching <code>${GP_req_missing}</code>"
+CoDeList=${CoDeList}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}")
+
+success_msg="found message(s) matching <code>${GP_no_missing}</code>"
+failure_msg="no message matching <code>${GP_no_missing}</code>"
+CoDeList=${CoDeList}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
+
+success_msg="found message matching <code>${GP_tgz_created}</code>"
+failure_msg="no message matching <code>${GP_tgz_created}</code>"
+CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
+
+comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}"
+
+
+# first construct comment_artefacts_list, abbreviated CoArList
+# then use it to set comment_artefacts
+CoArList=""
+
+# TARBALL should only contain a single tarball
+if [[ ! -z ${TARBALL} ]]; then
+    # Example of the detailed information for a tarball. The actual result MUST be a
+    # single line (no '\n') or it would break the structure of the markdown table
+    # that holds status updates of a bot job.
+    # 
+    # <dd>
+    #   <details>
+    #     <summary><code>eessi-2023.06-software-linux-x86_64-generic-1682696567.tar.gz</code></summary>
+    #     size: 234 MiB (245366784 bytes)<br/>
+    #     entries: 1234<br/>
+    #     modules under _2023.06/software/linux/x86_64/intel/cascadelake/modules/all/_<br/>
+    #     <pre>
+    #       GCC/9.3.0.lua<br/>
+    #       GCC/10.3.0.lua<br/>
+    #       OpenSSL/1.1.lua
+    #     </pre>
+    #     software under _2023.06/software/linux/x86_64/intel/cascadelake/software/_
+    #     <pre>
+    #       GCC/9.3.0/<br/>
+    #       CMake/3.20.1-GCCcore-10.3.0/<br/>
+    #       OpenMPI/4.1.1-GCC-10.3.0/
+    #     </pre>
+    #     other under _2023.06/software/linux/x86_64/intel/cascadelake/_
+    #     <pre>
+    #       .lmod/cache/spiderT.lua<br/>
+    #       .lmod/cache/spiderT.luac_5.1<br/>
+    #       .lmod/cache/timestamp
+    #     </pre>
+    #   </details>
+    # </dd>
+    size="$(stat --dereference --printf=%s ${TARBALL})"
+    size_mib=$((${size} >> 20))
+    tmpfile=$(mktemp --tmpdir=. tarfiles.XXXX)
+    tar tf ${TARBALL} > ${tmpfile}
+    entries=$(cat ${tmpfile} | wc -l)
+    # determine prefix from job config: VERSION/software/OS_TYPE/CPU_FAMILY/ARCHITECTURE
+    # e.g., 2023.06/software/linux/x86_64/intel/skylake_avx512
+    # cfg/job.cfg contains (only the attributes to be used are shown below):
+    # [repository]
+    # repo_version = 2023.06
+    # [architecture]
+    # os_type = linux
+    # software_subdir = x86_64/intel/skylake_avx512
+    repo_version=$(cfg_get_value "repository" "repo_version")
+    os_type=$(cfg_get_value "architecture" "os_type")
+    software_subdir=$(cfg_get_value "architecture" "software_subdir")
+    prefix="${repo_version}/software/${os_type}/${software_subdir}"
+
+    # extract directories/entries from tarball content
+    modules_entries=$(grep "${prefix}/modules" ${tmpfile})
+    software_entries=$(grep "${prefix}/software" ${tmpfile})
+    other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software")
+    other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u)
+    modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u)
+    software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u)
+
+    artefact_summary="<summary>$(print_code_item '__ITEM__' ${TARBALL})</summary>"
+    CoArList=""
+    CoArList="${CoArList}$(print_br_item2 'size: __ITEM__ MiB (__ITEM2__ bytes)' ${size_mib} ${size})"
+    CoArList="${CoArList}$(print_br_item 'entries: __ITEM__' ${entries})"
+    CoArList="${CoArList}$(print_br_item 'modules under ___ITEM___' ${prefix}/modules/all)"
+    CoArList="${CoArList}<pre>"
+    if [[ ! -z ${modules} ]]; then
+        while IFS= read -r mod ; do
+            CoArList="${CoArList}$(print_br_item '<code>__ITEM__</code>' ${mod})"
+        done <<< "${modules}"
+    else
+        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no module files in tarball')"
+    fi
+    CoArList="${CoArList}</pre>"
+    CoArList="${CoArList}$(print_br_item 'software under ___ITEM___' ${prefix}/software)"
+    CoArList="${CoArList}<pre>"
+    if [[ ! -z ${software_pkgs} ]]; then
+        while IFS= read -r sw_pkg ; do
+            CoArList="${CoArList}$(print_br_item '<code>__ITEM__</code>' ${sw_pkg})"
+        done <<< "${software_pkgs}"
+    else
+        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no software packages in tarball')"
+    fi
+    CoArList="${CoArList}</pre>"
+    CoArList="${CoArList}$(print_br_item 'other under ___ITEM___' ${prefix})"
+    CoArList="${CoArList}<pre>"
+    if [[ ! -z ${other_shortened} ]]; then
+        while IFS= read -r other ; do
+            CoArList="${CoArList}$(print_br_item '<code>__ITEM__</code>' ${other})"
+        done <<< "${other_shortened}"
+    else
+        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no other files in tarball')"
+    fi
+    CoArList="${CoArList}</pre>"
+else
+    CoArList="${CoArList}$(print_dd_item 'No artefacts were created or found.' '')"
+fi
+
+comment_artefacts_details="${comment_artefact_details_fmt/__ARTEFACT_SUMMARY__/${artefact_summary}}"
+comment_artefacts_details="${comment_artefacts_details/__ARTEFACT_DETAILS__/${CoArList}}"
+comment_artefacts="${comment_artefacts_fmt/__ARTEFACTS_LIST__/${comment_artefacts_details}}"
+
+# now put all pieces together creating comment_details from comment_template
+comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}}
+comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}
+comment_description=${comment_description/__ARTEFACTS_FMT__/${comment_artefacts}}
+
+echo "${comment_description}" >> ${job_result_file}
+
+# add overall result: SUCCESS, FAILURE, UNKNOWN + artefacts
+# - this should make use of subsequent steps such as deploying a tarball more
+#   efficient
+echo "status = ${status}" >> ${job_result_file}
+echo "artefacts = " >> ${job_result_file}
+echo "${TARBALL}" | sed -e 's/^/    /g' >> ${job_result_file}
+
+# remove tmpfile
+if [[ -f ${tmpfile} ]]; then
+    rm ${tmpfile}
+fi
+
+# exit script with value that reflects overall job result: SUCCESS (0), FAILURE (1)
+test "${status}" == "SUCCESS"
+exit $?
diff --git a/bot/check-result.sh b/bot/check-result.sh
new file mode 120000
index 0000000000..02f753db50
--- /dev/null
+++ b/bot/check-result.sh
@@ -0,0 +1 @@
+check-build.sh
\ No newline at end of file

From 7db0616a0c78f40864507d77085817af6eb7028a Mon Sep 17 00:00:00 2001
From: Alan O'Cais <alan.ocais@cecam.org>
Date: Fri, 1 Sep 2023 11:58:10 +0200
Subject: [PATCH 146/157] Remove need to pass argument to function

---
 init/eessi_archdetect.sh | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/init/eessi_archdetect.sh b/init/eessi_archdetect.sh
index 97b45bf936..58dd99eb6b 100755
--- a/init/eessi_archdetect.sh
+++ b/init/eessi_archdetect.sh
@@ -3,6 +3,8 @@ VERSION="1.1.0"
 
 # Logging
 LOG_LEVEL="INFO"
+# Default result type is a best match
+CPUPATH_RESULT="best"
 
 timestamp () {
     date "+%Y-%m-%d %H:%M:%S"
@@ -120,10 +122,11 @@ cpupath(){
         fi
     done
   
-    log "INFO" "cpupath: best match for host CPU: $best_arch_match"
-    if [ "allx" == "${1}x" ]; then
+    if [ "allx" == "${CPUPATH_RESULT}x" ]; then
+        log "INFO" "cpupath: all matches for host CPU: $all_arch_matches"
         echo "$all_arch_matches"
     else
+        log "INFO" "cpupath: best match for host CPU: $best_arch_match"
         echo "$best_arch_match"
     fi
 }
@@ -136,7 +139,7 @@ while getopts 'hdva' OPTION; do
         h) echo "$USAGE"; exit 0;;
         d) LOG_LEVEL="DEBUG";;
         v) echo "eessi_archdetect.sh v$VERSION"; exit 0;;
-        a) all="all";;
+        a) CPUPATH_RESULT="all";;
         ?) echo "$USAGE"; exit 1;;
     esac
 done
@@ -145,6 +148,6 @@ shift "$(($OPTIND -1))"
 ARGUMENT=${1:-none}
 
 case "$ARGUMENT" in
-    "cpupath") cpupath $all; exit;;
+    "cpupath") cpupath; exit;;
     *) echo "$USAGE"; log "ERROR" "Missing <action> argument (possible actions: 'cpupath')";;
 esac

From b81b729f9350d10dee15a3d561417c688258e99d Mon Sep 17 00:00:00 2001
From: Alan O'Cais <alan.ocais@cecam.org>
Date: Sun, 3 Sep 2023 16:40:32 +0200
Subject: [PATCH 147/157] Make sure all the directories actually exist

---
 .github/workflows/tests_archdetect.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml
index cabbaedb6d..676a7a8c4d 100644
--- a/.github/workflows/tests_archdetect.yml
+++ b/.github/workflows/tests_archdetect.yml
@@ -23,6 +23,9 @@ jobs:
       uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
     - name: test eessi_archdetect.sh
+      uses: eessi/github-action-eessi@main
+      with:
+        eessi_stack_version: '2023.06'
       run: |
           export EESSI_MACHINE_TYPE=${{matrix.proc_cpuinfo}}
           export EESSI_MACHINE_TYPE=${EESSI_MACHINE_TYPE%%/*}
@@ -41,3 +44,7 @@ jobs:
               echo "Test for ${{matrix.proc_cpuinfo}} FAILED: $CPU_ARCHES" >&2
               exit 1
           fi
+          # Check all those architectures exist
+          for dir in $(echo "$CPU_ARCHES" | tr ':' '\n'); do
+          [ -d "$EESSI_PREFIX/software/linux/$dir" ]
+          done

From 739a92da2dfabe70da52ae433c5152337effa494 Mon Sep 17 00:00:00 2001
From: Alan O'Cais <alan.ocais@cecam.org>
Date: Sun, 3 Sep 2023 16:44:16 +0200
Subject: [PATCH 148/157] Fix steps in action

---
 .github/workflows/tests_archdetect.yml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml
index 676a7a8c4d..92907607c4 100644
--- a/.github/workflows/tests_archdetect.yml
+++ b/.github/workflows/tests_archdetect.yml
@@ -21,11 +21,10 @@ jobs:
     steps:
     - name: checkout
       uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
-
-    - name: test eessi_archdetect.sh
-      uses: eessi/github-action-eessi@main
+    - uses: eessi/github-action-eessi@main
       with:
         eessi_stack_version: '2023.06'
+    - name: test eessi_archdetect.sh
       run: |
           export EESSI_MACHINE_TYPE=${{matrix.proc_cpuinfo}}
           export EESSI_MACHINE_TYPE=${EESSI_MACHINE_TYPE%%/*}
@@ -44,7 +43,7 @@ jobs:
               echo "Test for ${{matrix.proc_cpuinfo}} FAILED: $CPU_ARCHES" >&2
               exit 1
           fi
-          # Check all those architectures exist
+          # Check all those architectures actually exist
           for dir in $(echo "$CPU_ARCHES" | tr ':' '\n'); do
           [ -d "$EESSI_PREFIX/software/linux/$dir" ]
           done

From 45dc7dadd07ec5bdc4bb8f29b43d3eb75f5f9c34 Mon Sep 17 00:00:00 2001
From: Alan O'Cais <alan.ocais@cecam.org>
Date: Sun, 3 Sep 2023 16:57:17 +0200
Subject: [PATCH 149/157] Fix ARM detection for archdetect

---
 .github/workflows/tests_archdetect.yml                    | 6 +++---
 init/arch_specs/eessi_arch_arm.spec                       | 8 ++++----
 .../arm/neoverse-n1/AWS-awslinux-graviton2.all.output     | 1 -
 .../aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.output | 1 -
 .../arm/neoverse-n1/Azure-Ubuntu20-Altra.all.output       | 1 -
 .../aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.output   | 1 -
 .../arm/neoverse-v1/AWS-awslinux-graviton3.all.output     | 1 -
 .../aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.output | 1 -
 .../arm/neoverse_n1/AWS-awslinux-graviton2.all.output     | 1 +
 .../AWS-awslinux-graviton2.cpuinfo                        | 0
 .../aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.output | 1 +
 .../arm/neoverse_n1/Azure-Ubuntu20-Altra.all.output       | 1 +
 .../Azure-Ubuntu20-Altra.cpuinfo                          | 0
 .../aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.output   | 1 +
 .../arm/neoverse_v1/AWS-awslinux-graviton3.all.output     | 1 +
 .../AWS-awslinux-graviton3.cpuinfo                        | 0
 .../aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.output | 1 +
 17 files changed, 13 insertions(+), 13 deletions(-)
 delete mode 100644 tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.all.output
 delete mode 100644 tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.output
 delete mode 100644 tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.all.output
 delete mode 100644 tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.output
 delete mode 100644 tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.all.output
 delete mode 100644 tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.output
 create mode 100644 tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.all.output
 rename tests/archdetect/aarch64/arm/{neoverse-n1 => neoverse_n1}/AWS-awslinux-graviton2.cpuinfo (100%)
 create mode 100644 tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.output
 create mode 100644 tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.all.output
 rename tests/archdetect/aarch64/arm/{neoverse-n1 => neoverse_n1}/Azure-Ubuntu20-Altra.cpuinfo (100%)
 create mode 100644 tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.output
 create mode 100644 tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.all.output
 rename tests/archdetect/aarch64/arm/{neoverse-v1 => neoverse_v1}/AWS-awslinux-graviton3.cpuinfo (100%)
 create mode 100644 tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.output

diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml
index 92907607c4..76cc7315c7 100644
--- a/.github/workflows/tests_archdetect.yml
+++ b/.github/workflows/tests_archdetect.yml
@@ -14,9 +14,9 @@ jobs:
           - x86_64/amd/zen2/Azure-CentOS7-7V12
           - x86_64/amd/zen3/Azure-CentOS7-7V73X
           - ppc64le/power9le/unknown-power9le
-          - aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra
-          - aarch64/arm/neoverse-n1/AWS-awslinux-graviton2
-          - aarch64/arm/neoverse-v1/AWS-awslinux-graviton3
+          - aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra
+          - aarch64/arm/neoverse_n1/AWS-awslinux-graviton2
+          - aarch64/arm/neoverse_v1/AWS-awslinux-graviton3
       fail-fast: false
     steps:
     - name: checkout
diff --git a/init/arch_specs/eessi_arch_arm.spec b/init/arch_specs/eessi_arch_arm.spec
index 92f32a76d8..97d9cd2edc 100755
--- a/init/arch_specs/eessi_arch_arm.spec
+++ b/init/arch_specs/eessi_arch_arm.spec
@@ -1,6 +1,6 @@
 # ARM CPU architecture specifications
 # Software path in EESSI 	| Vendor ID 	| List of defining CPU features
-"aarch64/arm/neoverse-n1"	"ARM"		"asimd"		# Ampere Altra
-"aarch64/arm/neoverse-n1"	""		"asimd"		# AWS Graviton2
-"aarch64/arm/neoverse-v1"	"ARM"		"asimd svei8mm" 
-"aarch64/arm/neoverse-v1"	""		"asimd svei8mm"	# AWS Graviton3
+"aarch64/arm/neoverse_n1"	"ARM"		"asimd"		# Ampere Altra
+"aarch64/arm/neoverse_n1"	""		"asimd"		# AWS Graviton2
+"aarch64/arm/neoverse_v1"	"ARM"		"asimd svei8mm"
+"aarch64/arm/neoverse_v1"	""		"asimd svei8mm"	# AWS Graviton3
diff --git a/tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.all.output b/tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.all.output
deleted file mode 100644
index 4e4f20c1de..0000000000
--- a/tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.all.output
+++ /dev/null
@@ -1 +0,0 @@
-aarch64/arm/neoverse-n1:aarch64/generic
\ No newline at end of file
diff --git a/tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.output b/tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.output
deleted file mode 100644
index b4dc5e9f1b..0000000000
--- a/tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.output
+++ /dev/null
@@ -1 +0,0 @@
-aarch64/arm/neoverse-n1
diff --git a/tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.all.output b/tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.all.output
deleted file mode 100644
index 4e4f20c1de..0000000000
--- a/tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.all.output
+++ /dev/null
@@ -1 +0,0 @@
-aarch64/arm/neoverse-n1:aarch64/generic
\ No newline at end of file
diff --git a/tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.output b/tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.output
deleted file mode 100644
index b4dc5e9f1b..0000000000
--- a/tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.output
+++ /dev/null
@@ -1 +0,0 @@
-aarch64/arm/neoverse-n1
diff --git a/tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.all.output b/tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.all.output
deleted file mode 100644
index 61f4d892f0..0000000000
--- a/tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.all.output
+++ /dev/null
@@ -1 +0,0 @@
-aarch64/arm/neoverse-v1:aarch64/arm/neoverse-n1:aarch64/generic
\ No newline at end of file
diff --git a/tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.output b/tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.output
deleted file mode 100644
index 20db96d01f..0000000000
--- a/tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.output
+++ /dev/null
@@ -1 +0,0 @@
-aarch64/arm/neoverse-v1
diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.all.output b/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.all.output
new file mode 100644
index 0000000000..e6aee274ae
--- /dev/null
+++ b/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.all.output
@@ -0,0 +1 @@
+aarch64/arm/neoverse_n1:aarch64/generic
\ No newline at end of file
diff --git a/tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.cpuinfo b/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.cpuinfo
similarity index 100%
rename from tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.cpuinfo
rename to tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.cpuinfo
diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.output b/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.output
new file mode 100644
index 0000000000..c841d54b50
--- /dev/null
+++ b/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.output
@@ -0,0 +1 @@
+aarch64/arm/neoverse_n1
diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.all.output b/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.all.output
new file mode 100644
index 0000000000..e6aee274ae
--- /dev/null
+++ b/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.all.output
@@ -0,0 +1 @@
+aarch64/arm/neoverse_n1:aarch64/generic
\ No newline at end of file
diff --git a/tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.cpuinfo b/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.cpuinfo
similarity index 100%
rename from tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.cpuinfo
rename to tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.cpuinfo
diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.output b/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.output
new file mode 100644
index 0000000000..c841d54b50
--- /dev/null
+++ b/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.output
@@ -0,0 +1 @@
+aarch64/arm/neoverse_n1
diff --git a/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.all.output b/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.all.output
new file mode 100644
index 0000000000..bda6c2e498
--- /dev/null
+++ b/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.all.output
@@ -0,0 +1 @@
+aarch64/arm/neoverse_v1:aarch64/arm/neoverse_n1:aarch64/generic
\ No newline at end of file
diff --git a/tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.cpuinfo b/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.cpuinfo
similarity index 100%
rename from tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.cpuinfo
rename to tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.cpuinfo
diff --git a/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.output b/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.output
new file mode 100644
index 0000000000..0c80cb1efb
--- /dev/null
+++ b/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.output
@@ -0,0 +1 @@
+aarch64/arm/neoverse_v1

From 71cf7c95021ab16b65f098b41af853b1534aa62c Mon Sep 17 00:00:00 2001
From: Alan O'Cais <alan.ocais@cecam.org>
Date: Sun, 3 Sep 2023 17:01:13 +0200
Subject: [PATCH 150/157] Fix ARM spec

---
 init/arch_specs/eessi_arch_arm.spec | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/init/arch_specs/eessi_arch_arm.spec b/init/arch_specs/eessi_arch_arm.spec
index 97d9cd2edc..b5c9275043 100755
--- a/init/arch_specs/eessi_arch_arm.spec
+++ b/init/arch_specs/eessi_arch_arm.spec
@@ -1,6 +1,6 @@
 # ARM CPU architecture specifications
 # Software path in EESSI 	| Vendor ID 	| List of defining CPU features
-"aarch64/arm/neoverse_n1"	"ARM"		"asimd"		# Ampere Altra
-"aarch64/arm/neoverse_n1"	""		"asimd"		# AWS Graviton2
-"aarch64/arm/neoverse_v1"	"ARM"		"asimd svei8mm"
-"aarch64/arm/neoverse_v1"	""		"asimd svei8mm"	# AWS Graviton3
+"aarch64/neoverse_n1"	"ARM"		"asimd"		# Ampere Altra
+"aarch64/neoverse_n1"	""		"asimd"		# AWS Graviton2
+"aarch64/neoverse_v1"	"ARM"		"asimd svei8mm"
+"aarch64/neoverse_v1"	""		"asimd svei8mm"	# AWS Graviton3

From 98a95feaaee00046ec233ca92f37e7862d6b3035 Mon Sep 17 00:00:00 2001
From: Alan O'Cais <alan.ocais@cecam.org>
Date: Sun, 3 Sep 2023 17:05:15 +0200
Subject: [PATCH 151/157] Also fix archdetect tests

---
 .../aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.all.output   | 2 +-
 .../aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.output       | 2 +-
 .../aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.all.output     | 2 +-
 .../aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.output         | 2 +-
 .../aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.all.output   | 2 +-
 .../aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.output       | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.all.output b/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.all.output
index e6aee274ae..340aaa5d02 100644
--- a/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.all.output
+++ b/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.all.output
@@ -1 +1 @@
-aarch64/arm/neoverse_n1:aarch64/generic
\ No newline at end of file
+aarch64/neoverse_n1:aarch64/generic
diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.output b/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.output
index c841d54b50..a9bd49c75c 100644
--- a/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.output
+++ b/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.output
@@ -1 +1 @@
-aarch64/arm/neoverse_n1
+aarch64/neoverse_n1
diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.all.output b/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.all.output
index e6aee274ae..340aaa5d02 100644
--- a/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.all.output
+++ b/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.all.output
@@ -1 +1 @@
-aarch64/arm/neoverse_n1:aarch64/generic
\ No newline at end of file
+aarch64/neoverse_n1:aarch64/generic
diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.output b/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.output
index c841d54b50..a9bd49c75c 100644
--- a/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.output
+++ b/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.output
@@ -1 +1 @@
-aarch64/arm/neoverse_n1
+aarch64/neoverse_n1
diff --git a/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.all.output b/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.all.output
index bda6c2e498..920d5f9996 100644
--- a/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.all.output
+++ b/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.all.output
@@ -1 +1 @@
-aarch64/arm/neoverse_v1:aarch64/arm/neoverse_n1:aarch64/generic
\ No newline at end of file
+aarch64/neoverse_v1:aarch64/neoverse_n1:aarch64/generic
diff --git a/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.output b/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.output
index 0c80cb1efb..a8e072a9c6 100644
--- a/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.output
+++ b/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.output
@@ -1 +1 @@
-aarch64/arm/neoverse_v1
+aarch64/neoverse_v1

From ef1a53aff3b22b120a0405aaaa0521bb6426f219 Mon Sep 17 00:00:00 2001
From: Alan O'Cais <alan.ocais@cecam.org>
Date: Sun, 3 Sep 2023 17:20:00 +0200
Subject: [PATCH 152/157] Make tests flexible in terms of EESSI version

---
 .github/workflows/tests_archdetect.yml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml
index 76cc7315c7..b6fc6434b2 100644
--- a/.github/workflows/tests_archdetect.yml
+++ b/.github/workflows/tests_archdetect.yml
@@ -21,9 +21,7 @@ jobs:
     steps:
     - name: checkout
       uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
-    - uses: eessi/github-action-eessi@main
-      with:
-        eessi_stack_version: '2023.06'
+    - uses: eessi/github-action-eessi@v2
     - name: test eessi_archdetect.sh
       run: |
           export EESSI_MACHINE_TYPE=${{matrix.proc_cpuinfo}}
@@ -45,5 +43,6 @@ jobs:
           fi
           # Check all those architectures actually exist
           for dir in $(echo "$CPU_ARCHES" | tr ':' '\n'); do
-          [ -d "$EESSI_PREFIX/software/linux/$dir" ]
+              # Search all EESSI versions as we may drop support at some point 
+              ls -d "$EESSI_PREFIX/../*/software/linux/$dir" 
           done

From 5665fad15dea939751ad6eb17286a4670353f973 Mon Sep 17 00:00:00 2001
From: Alan O'Cais <alan.ocais@cecam.org>
Date: Sun, 3 Sep 2023 17:28:38 +0200
Subject: [PATCH 153/157] Fix hash of EESSI action

---
 .github/workflows/tests_archdetect.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml
index b6fc6434b2..f5781ef2a7 100644
--- a/.github/workflows/tests_archdetect.yml
+++ b/.github/workflows/tests_archdetect.yml
@@ -21,7 +21,9 @@ jobs:
     steps:
     - name: checkout
       uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
-    - uses: eessi/github-action-eessi@v2
+    - uses: eessi/github-action-eessi@58b50fd2eead2162c2b9ac258d4fb60cc9f30503 # v2.0.13
+      with:
+        eessi_stack_version: '2023.06'
     - name: test eessi_archdetect.sh
       run: |
           export EESSI_MACHINE_TYPE=${{matrix.proc_cpuinfo}}

From 07bfbb1d9b14bdb971d82b7a6d61c7e9818d8946 Mon Sep 17 00:00:00 2001
From: Alan O'Cais <alan.ocais@cecam.org>
Date: Sun, 3 Sep 2023 17:32:41 +0200
Subject: [PATCH 154/157] Careful with quotes

---
 .github/workflows/tests_archdetect.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml
index f5781ef2a7..a794e5ebc4 100644
--- a/.github/workflows/tests_archdetect.yml
+++ b/.github/workflows/tests_archdetect.yml
@@ -21,7 +21,8 @@ jobs:
     steps:
     - name: checkout
       uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
-    - uses: eessi/github-action-eessi@58b50fd2eead2162c2b9ac258d4fb60cc9f30503 # v2.0.13
+    - name: Enable EESSI
+      uses: eessi/github-action-eessi@58b50fd2eead2162c2b9ac258d4fb60cc9f30503 # v2.0.13
       with:
         eessi_stack_version: '2023.06'
     - name: test eessi_archdetect.sh
@@ -46,5 +47,5 @@ jobs:
           # Check all those architectures actually exist
           for dir in $(echo "$CPU_ARCHES" | tr ':' '\n'); do
               # Search all EESSI versions as we may drop support at some point 
-              ls -d "$EESSI_PREFIX/../*/software/linux/$dir" 
+              ls -d "$EESSI_PREFIX"/../*/software/linux/"$dir" 
           done

From b10415b16834b15f17a49ade9391d84f1fc2d469 Mon Sep 17 00:00:00 2001
From: Alan O'Cais <alan.ocais@cecam.org>
Date: Sun, 3 Sep 2023 17:43:46 +0200
Subject: [PATCH 155/157] Also drop arm subdir from tests

---
 .github/workflows/tests_archdetect.yml                      | 6 +++---
 .../{arm => }/neoverse_n1/AWS-awslinux-graviton2.all.output | 0
 .../{arm => }/neoverse_n1/AWS-awslinux-graviton2.cpuinfo    | 0
 .../{arm => }/neoverse_n1/AWS-awslinux-graviton2.output     | 0
 .../{arm => }/neoverse_n1/Azure-Ubuntu20-Altra.all.output   | 0
 .../{arm => }/neoverse_n1/Azure-Ubuntu20-Altra.cpuinfo      | 0
 .../{arm => }/neoverse_n1/Azure-Ubuntu20-Altra.output       | 0
 .../{arm => }/neoverse_v1/AWS-awslinux-graviton3.all.output | 0
 .../{arm => }/neoverse_v1/AWS-awslinux-graviton3.cpuinfo    | 0
 .../{arm => }/neoverse_v1/AWS-awslinux-graviton3.output     | 0
 10 files changed, 3 insertions(+), 3 deletions(-)
 rename tests/archdetect/aarch64/{arm => }/neoverse_n1/AWS-awslinux-graviton2.all.output (100%)
 rename tests/archdetect/aarch64/{arm => }/neoverse_n1/AWS-awslinux-graviton2.cpuinfo (100%)
 rename tests/archdetect/aarch64/{arm => }/neoverse_n1/AWS-awslinux-graviton2.output (100%)
 rename tests/archdetect/aarch64/{arm => }/neoverse_n1/Azure-Ubuntu20-Altra.all.output (100%)
 rename tests/archdetect/aarch64/{arm => }/neoverse_n1/Azure-Ubuntu20-Altra.cpuinfo (100%)
 rename tests/archdetect/aarch64/{arm => }/neoverse_n1/Azure-Ubuntu20-Altra.output (100%)
 rename tests/archdetect/aarch64/{arm => }/neoverse_v1/AWS-awslinux-graviton3.all.output (100%)
 rename tests/archdetect/aarch64/{arm => }/neoverse_v1/AWS-awslinux-graviton3.cpuinfo (100%)
 rename tests/archdetect/aarch64/{arm => }/neoverse_v1/AWS-awslinux-graviton3.output (100%)

diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml
index a794e5ebc4..61bf0e73fd 100644
--- a/.github/workflows/tests_archdetect.yml
+++ b/.github/workflows/tests_archdetect.yml
@@ -14,9 +14,9 @@ jobs:
           - x86_64/amd/zen2/Azure-CentOS7-7V12
           - x86_64/amd/zen3/Azure-CentOS7-7V73X
           - ppc64le/power9le/unknown-power9le
-          - aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra
-          - aarch64/arm/neoverse_n1/AWS-awslinux-graviton2
-          - aarch64/arm/neoverse_v1/AWS-awslinux-graviton3
+          - aarch64/neoverse_n1/Azure-Ubuntu20-Altra
+          - aarch64/neoverse_n1/AWS-awslinux-graviton2
+          - aarch64/neoverse_v1/AWS-awslinux-graviton3
       fail-fast: false
     steps:
     - name: checkout
diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.all.output b/tests/archdetect/aarch64/neoverse_n1/AWS-awslinux-graviton2.all.output
similarity index 100%
rename from tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.all.output
rename to tests/archdetect/aarch64/neoverse_n1/AWS-awslinux-graviton2.all.output
diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.cpuinfo b/tests/archdetect/aarch64/neoverse_n1/AWS-awslinux-graviton2.cpuinfo
similarity index 100%
rename from tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.cpuinfo
rename to tests/archdetect/aarch64/neoverse_n1/AWS-awslinux-graviton2.cpuinfo
diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.output b/tests/archdetect/aarch64/neoverse_n1/AWS-awslinux-graviton2.output
similarity index 100%
rename from tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.output
rename to tests/archdetect/aarch64/neoverse_n1/AWS-awslinux-graviton2.output
diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.all.output b/tests/archdetect/aarch64/neoverse_n1/Azure-Ubuntu20-Altra.all.output
similarity index 100%
rename from tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.all.output
rename to tests/archdetect/aarch64/neoverse_n1/Azure-Ubuntu20-Altra.all.output
diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.cpuinfo b/tests/archdetect/aarch64/neoverse_n1/Azure-Ubuntu20-Altra.cpuinfo
similarity index 100%
rename from tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.cpuinfo
rename to tests/archdetect/aarch64/neoverse_n1/Azure-Ubuntu20-Altra.cpuinfo
diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.output b/tests/archdetect/aarch64/neoverse_n1/Azure-Ubuntu20-Altra.output
similarity index 100%
rename from tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.output
rename to tests/archdetect/aarch64/neoverse_n1/Azure-Ubuntu20-Altra.output
diff --git a/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.all.output b/tests/archdetect/aarch64/neoverse_v1/AWS-awslinux-graviton3.all.output
similarity index 100%
rename from tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.all.output
rename to tests/archdetect/aarch64/neoverse_v1/AWS-awslinux-graviton3.all.output
diff --git a/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.cpuinfo b/tests/archdetect/aarch64/neoverse_v1/AWS-awslinux-graviton3.cpuinfo
similarity index 100%
rename from tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.cpuinfo
rename to tests/archdetect/aarch64/neoverse_v1/AWS-awslinux-graviton3.cpuinfo
diff --git a/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.output b/tests/archdetect/aarch64/neoverse_v1/AWS-awslinux-graviton3.output
similarity index 100%
rename from tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.output
rename to tests/archdetect/aarch64/neoverse_v1/AWS-awslinux-graviton3.output

From 409ba53ba360268449cffb3de59a91700eb1f78f Mon Sep 17 00:00:00 2001
From: Alan O'Cais <alan.ocais@cecam.org>
Date: Sun, 3 Sep 2023 17:45:24 +0200
Subject: [PATCH 156/157] Remove EESSI version selection

---
 .github/workflows/tests_archdetect.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml
index 61bf0e73fd..37338693c5 100644
--- a/.github/workflows/tests_archdetect.yml
+++ b/.github/workflows/tests_archdetect.yml
@@ -23,8 +23,6 @@ jobs:
       uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
     - name: Enable EESSI
       uses: eessi/github-action-eessi@58b50fd2eead2162c2b9ac258d4fb60cc9f30503 # v2.0.13
-      with:
-        eessi_stack_version: '2023.06'
     - name: test eessi_archdetect.sh
       run: |
           export EESSI_MACHINE_TYPE=${{matrix.proc_cpuinfo}}

From 92b764998fc46b0906b20de7346d0044248281b4 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 5 Oct 2023 10:59:49 +0200
Subject: [PATCH 157/157] remove bot configuration (main)

---
 bot/bot-eessi-aws-citc.cfg | 151 -------------------------------------
 1 file changed, 151 deletions(-)
 delete mode 100644 bot/bot-eessi-aws-citc.cfg

diff --git a/bot/bot-eessi-aws-citc.cfg b/bot/bot-eessi-aws-citc.cfg
deleted file mode 100644
index 5b3ad34612..0000000000
--- a/bot/bot-eessi-aws-citc.cfg
+++ /dev/null
@@ -1,151 +0,0 @@
-# Also see documentation at https://github.com/EESSI/eessi-bot-software-layer/blob/main/README.md#step5.5
-
-[github]
-# replace '123456' with the ID of your GitHub App
-app_id = 281041
-
-# a short (!) name for your app instance that can be used for example
-#   when adding/updating a comment to a PR
-# (!) a short yet descriptive name is preferred because it appears in
-#   comments to the PR
-# for example, the name could include the name of the cluster the bot
-#   runs on and the username which runs the bot
-# NOTE avoid putting an actual username here as it will be visible on
-#      potentially publicly accessible GitHub pages.
-app_name = eessi-bot-citc-aws
-
-# replace '12345678' with the ID of the installation of your GitHub App
-#   (can be derived by creating an event and then checking for the list
-#   of sent events and its payload either via the Smee channel's web page
-#   or via the Advanced section of your GitHub App on github.com)
-installation_id = 33078935
-
-# path to the private key that was generated when the GitHub App was registered
-private_key = /mnt/shared/home/bot/eessi-bot-software-layer/eessi-bot-citc-aws-private-key.pem
-
-
-[buildenv]
-# name of the job script used for building an EESSI stack
-build_job_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/bot-build.slurm
-
-# The container_cachedir may be used to reuse downloaded container image files
-# across jobs. Thus, jobs can more quickly launch containers.
-container_cachedir = /mnt/shared/home/bot/eessi-bot-software-layer/containers-cache-dir
-
-# it may happen that we need to customize some CVMFS configuration
-#   the value of cvmfs_customizations is a dictionary which maps a file
-#   name to an entry that needs to be added to that file
-cvmfs_customizations = {}
-
-# if compute nodes have no internet connection, we need to set http(s)_proxy
-#   or commands such as pip3 cannot download software from package repositories
-#   for example, the temporary EasyBuild is installed via pip3 first
-# http_proxy = http://PROXY_DNS:3128/
-# https_proxy = http://PROXY_DNS:3128/
-
-# directory under which the bot prepares directories per job
-#   structure created is as follows: YYYY.MM/pr_PR_NUMBER/event_EVENT_ID/run_RUN_NUMBER/OS+SUBDIR
-jobs_base_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs
-
-# configure environment
-#   list of comma-separated modules to be loaded by build_job_script
-#   useful/needed if some tool is not provided as system-wide package
-#   (read by bot and handed over to build_job_script via parameter
-#   --load-modules)
-load_modules =
-
-# PATH to temporary directory on build node ... ends up being used for
-#     for example, EESSI_TMPDIR --> /tmp/$USER/EESSI
-#   escaping variables with '\' delays expansion to the start of the
-#     build_job_script; this can be used for referencing environment
-#     variables that are only set inside a Slurm job
-local_tmp = /tmp/$USER/EESSI
-
-# parameters to be added to all job submissions
-# NOTE do not quote parameter string. Quotes are retained when reading in config and
-#      then the whole 'string' is recognised as a single parameter.
-# NOTE 2 '--get-user-env' may be needed on systems where the job's environment needs
-#        to be initialised as if it is for a login shell.
-# note: hardcoded 24h time limit until https://github.com/EESSI/eessi-bot-software-layer/issues/146 is fixed
-slurm_params = --hold --time=24:0:0
-
-# full path to the job submission command
-submit_command = /usr/bin/sbatch
-
-# which GH account has the permission to trigger the build (by setting
-# the label 'bot:build' (apparently this cannot be restricted on GitHub)
-# if value is left/empty everyone can trigger the build
-# value can be a space delimited list of GH accounts
-build_permission = boegel trz42 bedroge
-
-[architecturetargets]
-# defines both for which architectures the bot will build
-#   and what submission parameters shall be used
-# medium instances (8 cores, 16GB RAM)
-#arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge"}
-# larger instances (16 cores, 32GB RAM)
-arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge"}
-
-[repo_targets]
-# defines for which repository a arch_target should be build for
-#
-# only building for repository EESSI-pilot
-repo_target_map = { "linux/x86_64/generic" : ["EESSI-pilot"], "linux/x86_64/intel/haswell" : ["EESSI-pilot"], "linux/x86_64/intel/skylake_avx512" : ["EESSI-pilot"], "linux/x86_64/amd/zen2": ["EESSI-pilot"], "linux/x86_64/amd/zen3" : ["EESSI-pilot"], "linux/aarch64/generic" : ["EESSI-pilot"], "linux/aarch64/graviton2" : ["EESSI-pilot"], "linux/aarch64/graviton3" : ["EESSI-pilot"]}
-
-# points to definition of repositories (default EESSI-pilot defined by build container)
-repos_cfg_dir = /mnt/shared/home/bot/eessi-bot-software-layer/cfg-bundles
-
-# configuration for event handler which receives events from a GitHub repository.
-[event_handler]
-# path to the log file to log messages for event handler
-log_path = /mnt/shared/home/bot/eessi-bot-software-layer/eessi_bot_event_handler.log
-
-
-[job_manager]
-# path to the log file to log messages for job manager
-log_path = /mnt/shared/home/bot/eessi-bot-software-layer/eessi_bot_job_manager.log
-
-# directory where job manager stores information about jobs to be tracked
-#   e.g. as symbolic link JOBID -> directory to job
-job_ids_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs
-
-# full path to the job status checking command
-poll_command = /usr/bin/squeue
-
-# polling interval in seconds
-poll_interval = 60
-
-# full path to the command for manipulating existing jobs
-scontrol_command = /usr/bin/scontrol
-
-[deploycfg]
-# script for uploading built software packages
-tarball_upload_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/eessi-upload-to-staging
-
-# URL to S3/minio bucket
-#   if attribute is set, bucket_base will be constructed as follows
-#     bucket_base=${endpoint_url}/${bucket_name}
-#   otherwise, bucket_base will be constructed as follows
-#     bucket_base=https://${bucket_name}.s3.amazonaws.com
-# - The former variant is used for non AWS S3 services, eg, minio, or when
-#   the bucket name is not provided in the hostname (see latter case).
-# - The latter variant is used for AWS S3 services.
-#endpoint_url = URL_TO_S3_SERVER
-
-# bucket name
-bucket_name = eessi-staging
-
-# upload policy: defines what policy is used for uploading built artefacts
-#                to an S3 bucket
-# 'all' ..: upload all artefacts (mulitple uploads of the same artefact possible)
-# 'latest': for each build target (eessi-VERSION-{software,init,compat}-OS-ARCH)
-#           only upload the latest built artefact
-# 'once'  : only once upload any built artefact for the build target
-# 'none'  : do not upload any built artefacts
-upload_policy = once
-
-# which GH account has the permission to trigger the deployment (by setting
-# the label 'bot:deploy' (apparently this cannot be restricted on GitHub)
-# if value is left/empty everyone can trigger the deployment
-# value can be a space delimited list of GH accounts
-deploy_permission = boegel trz42 bedroge