bgruening · pvanheus · Nov 15, 2021 · Nov 15, 2021 · Nov 18, 2021 · Nov 18, 2021
diff --git a/README.md b/README.md
@@ -1002,3 +1002,58 @@ You can file an [github issue](https://github.com/bgruening/docker-galaxy-stable
 us on the [Galaxy development list](http://lists.bx.psu.edu/listinfo/galaxy-dev).
 
 If you like this service please fill out this survey: https://www.surveymonkey.de/r/denbi-service?sc=rbc&tool=galaxy-docker
+
+### SANBI / COMBAT-SARS-CoV-2 specific notes
+
+#### Tool installation
+
+IRIDA plugins built with the [IRIDA plugin builder](https://github.com/COMBAT-TB/irida-plugin-builder) have a `tools.yaml`
+file in the same folder as their workflows (the `irida_workflow.xml` etc). This can be used, with [ephemeris](https://ephemeris.readthedocs.io/en/latest/) to install tools into Galaxy. Note while this will list tools for each workflow, it
+does not have info on data managers and those need to be listed and installed separately.
+
+#### Running Data Managers
+
+As admin you need to run the Data Manager for Galaxy manually. Connect to Galaxy on port 90 of the machine where IRIDA is
+installed and use the Admin menu (Local Data) to find and run the DM.
+
+#### Singularity images
+
+Singularity container images are managed through the [Container Resolvers](https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/config/sample/container_resolvers_conf.xml.sample) dependency resolvers. There are 3 approaches: 
+
+1. Explictely mentioned containers (these are mentioned in the tool XML)
+2. Mulled containers available on quay.io
+3. Containers build on the server
+
+Both (2) and (3) are designed to cache the containers on the server after download or building them. Neither approach works with docker-ized Galaxy at this stage. The cache is located, by default at `database/container_cache/singularity/mulled` in the `export/galaxy` directory. Pre-built images for the Oxford Nanopore pipeline can be fetched with this script:
+
+```
+#!/bin/bash
+
+set -e
+
+DEST=$(readlink -f $1)
+
+mkdir tmp
+cd tmp
+
+wget -R "index.html*" --no-host-directories --recursive --no-parent https://gdev.sanbi.ac.za/static/singularity_images/
+
+mv static/singularity_images/* $DEST
+
+cd ..
+rm -rf tmp
+```
+
+This downloads images and puts them in the directory specified - e.g. if this script is saved as `fetch_images.sh` it can be run as `fetch_images.sh dest` and it will put the images in a `dest` directory (this needs to exist before the script is run).
+
+#### Ensure that conda is not used
+
+Put this:
+
+```
+<dependency_resolvers>
+</dependency_resolvers>
+```
+
+in `galaxy-configurator/templates/galaxy/dependency_resolvers_conf.xml` to ensure that conda is not used for dependency
+resolution.
diff --git a/compose/.gitignore b/compose/.gitignore
@@ -0,0 +1 @@
+galaxy_tools.yml
diff --git a/compose/base-images/galaxy-container-base/Dockerfile b/compose/base-images/galaxy-container-base/Dockerfile
@@ -12,7 +12,7 @@ RUN apt update && apt install --no-install-recommends cryptsetup-bin uuid-dev -y
 
 ENV PATH=/usr/local/go/bin:${PATH}
 ENV SINGULARITY_VERSION=3.5.3
-RUN wget https://github.com/sylabs/singularity/releases/download/v${SINGULARITY_VERSION}/singularity-${SINGULARITY_VERSION}.tar.gz \
+RUN wget https://github.com/singularityware/singularity/releases/download/v${SINGULARITY_VERSION}/singularity-${SINGULARITY_VERSION}.tar.gz \
     && tar -xzf singularity-${SINGULARITY_VERSION}.tar.gz \
     && cd singularity \
     && ./mconfig \

diff --git a/compose/docker-compose.irida.yml b/compose/docker-compose.irida.yml
@@ -0,0 +1,55 @@
+# Install IRIDA
+version: "3.7"
+services:
+  galaxy-server:
+    environment:
+      - GALAXY_EXTRA_GROUP=root
+      - GALAXY_URL=http://nginx:90/
+    volumes:
+      - irida_web_data:/data/irida
+  galaxy-configurator:
+    environment:
+      - GALAXY_SINGULARITY_VOLUMES=/data/irida:rw
+      - GALAXY_CONFIG_ALLOW_PATH_PASTE=true
+      - IRIDA_OVERWRITE_CONFIG=true
+      - IRIDA_CONF_DIR=/etc/irida
+      - IRIDA_ENABLED=true
+    volumes:
+      - ${EXPORT_DIR:-./export}/irida:/etc/irida
+  nginx:
+    ports:
+      - 90:90
+  irida-web:
+    build: irida
+    container_name: irida-web
+    depends_on:
+      - irida-mysql
+    ports:
+      - "8080:8080"
+    volumes:
+      - ${EXPORT_DIR:-./export}/irida:/etc/irida
+      - irida_web_data:/data/irida
+    networks:
+      - galaxy
+
+  irida-mysql:
+    restart: always
+    image: mariadb:10.5.6
+    container_name: irida-mysql
+    environment:
+      MYSQL_ROOT_PASSWORD: rootpasswd
+      MYSQL_DATABASE: irida_db
+      MYSQL_USER: irida_user
+      MYSQL_PASSWORD: irida_passwd
+    volumes:
+      - irida_mysql_data:/var/lib/mysql
+    ports:
+      - "33060:3306"   
+    networks:
+      - galaxy
+
+volumes:
+  irida_mysql_data:
+  irida_web_data:
+  irida_galaxy_data:      
+
diff --git a/compose/docker-compose.irida_ssl.yml b/compose/docker-compose.irida_ssl.yml
@@ -0,0 +1,10 @@
+# Extend IRIDA to use SSL
+version: "3.7"
+services:
+  galaxy-configurator:
+    environment:
+      - IRIDA_SSL_ENABLED=true
+  nginx:
+    ports:
+      - 443:443
+
diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml
@@ -30,6 +30,7 @@ services:
     environment:
       - EXPORT_DIR=${EXPORT_DIR:-./export}
       - HOST_PWD=$PWD
+      - GALAXY_CONF_DIR=/galaxy/config
       - GALAXY_OVERWRITE_CONFIG=true
       - GALAXY_DEPENDENCY_RESOLUTION=conda
       - GALAXY_JOB_RUNNER=local
@@ -40,6 +41,7 @@ services:
       - GALAXY_CONFIG_AMQP_INTERNAL_CONNECTION=amqp://galaxy:vaiJa3ieghai2ief0jao@rabbitmq/galaxy
       - GALAXY_PROXY_PREFIX=${GALAXY_PROXY_PREFIX:-}
       - GALAXY_CONFIG_CLEANUP_JOB=onsuccess
+      - GALAXY_ENABLE_TESTTOOLSHED=true
       - NGINX_OVERWRITE_CONFIG=true
     volumes:
       - ${EXPORT_DIR:-./export}/galaxy/config:/galaxy/config
@@ -89,3 +91,6 @@ services:
       - galaxy
 networks:
   galaxy:
+    driver: bridge
+    driver_opts:
+      com.docker.network.driver.mtu: 1450
diff --git a/compose/extra-tools.yml b/compose/extra-tools.yml
@@ -0,0 +1,7 @@
+tools:
+  - name: "data_manager_primer_scheme_bedfiles"
+    owner: "iuc"
+    tool_shed_url: "toolshed.g2.bx.psu.edu"
+    revisions:
+      - "dd451e45681c"
+    tool_panel_section_label: "COMBAT-SARS-COV-2::ARTIC-MINION"
diff --git a/compose/galaxy-configurator/Dockerfile b/compose/galaxy-configurator/Dockerfile
@@ -1,6 +1,6 @@
 FROM alpine:3.11
 
-RUN apk add --no-cache bash python3 \
+RUN apk add --no-cache bash python3 rsync \
     && pip3 install j2cli[yaml] jinja2-ansible-filters
 
 COPY ./templates /templates

diff --git a/compose/galaxy-configurator/run.sh b/compose/galaxy-configurator/run.sh
@@ -6,10 +6,11 @@ export GALAXY_CONF_DIR=${GALAXY_CONF_DIR:-/galaxy/config} \
        SLURM_CONF_DIR=${SLURM_CONF_DIR:-/etc/slurm-llnl} \
        HTCONDOR_CONF_DIR=${HTCONDOR_CONF_DIR:-/htcondor} \
        PULSAR_CONF_DIR=${PULSAR_CONF_DIR:-/pulsar/config} \
-       KIND_CONF_DIR=${KIND_CONF_DIR:-/kind}
+       KIND_CONF_DIR=${KIND_CONF_DIR:-/kind} \
+       IRIDA_CONF_DIR=${IRIDA_CONF_DIR:-/etc/irida}
 
 echo "Locking all configurations"
-locks=("$GALAXY_CONF_DIR" "$SLURM_CONF_DIR" "$HTCONDOR_CONF_DIR" "$PULSAR_CONF_DIR" "$KIND_CONF_DIR")
+locks=("$GALAXY_CONF_DIR" "$SLURM_CONF_DIR" "$HTCONDOR_CONF_DIR" "$PULSAR_CONF_DIR" "$KIND_CONF_DIR" "$IRIDA_CONF_DIR")
 for lock in "${locks[@]}"; do
   echo "Locking $lock"
   touch "${lock}/configurator.lock"
@@ -111,6 +112,25 @@ else
   chmod a+r "${GALAXY_KUBECONFIG:-${KIND_CONF_DIR}/.kube/config_in_docker}"
 fi
 
+# IRIDA configuration
+if [ "$IRIDA_OVERWRITE_CONFIG" != "true" ]; then
+  echo "IRIDA_OVERWRITE_CONFIG is not true. Skipping configuration of IRIDA"
+else
+  irida_configs=( "irida.conf" "web.conf" "static" "templates" "plugins" )
+
+  for conf in "${irida_configs[@]}"; do
+    echo "Configuring $conf"
+    if [[ "$conf" == *.conf ]] ; then
+      j2 --customize /customize.py --undefined -o "/tmp/$conf" "/templates/irida/$conf.j2" /base_config.yml
+      echo "The following changes will be applied to $conf:"
+      diff "${IRIDA_CONF_DIR}/$conf" "/tmp/$conf"
+      mv -f "/tmp/$conf" "${IRIDA_CONF_DIR}/$conf"
+    else
+      rsync -aP --delete "/templates/irida/$conf" "${IRIDA_CONF_DIR}/"
+    fi
+  done
+fi
+
 echo "Releasing all locks (except Galaxy) if it didn't happen already"
 locks=("$SLURM_CONF_DIR" "$HTCONDOR_CONF_DIR" "$PULSAR_CONF_DIR" "$KIND_CONF_DIR")
 for lock in "${locks[@]}"; do
@@ -138,7 +158,7 @@ if [ ! -f /base_config.yml ]; then
   touch /base_config.yml
 fi
 
-galaxy_configs=( "job_conf.xml" "galaxy.yml" "job_metrics.xml" "container_resolvers_conf.xml" "GALAXY_PROXY_PREFIX.txt" )
+galaxy_configs=( "tool_sheds_conf.xml" "job_conf.xml" "galaxy.yml" "job_metrics.xml" "container_resolvers_conf.xml" "GALAXY_PROXY_PREFIX.txt" )
 
 for conf in "${galaxy_configs[@]}"; do
   echo "Configuring $conf"

diff --git a/compose/galaxy-configurator/templates/galaxy/container_resolvers_conf.xml.j2 b/compose/galaxy-configurator/templates/galaxy/container_resolvers_conf.xml.j2
@@ -17,7 +17,7 @@
        images to building local singularity images.
        -->
 
-  <mulled auto_install="True"/>
+  <mulled auto_install="False"/>
   <!-- mulled: only resolves container URI through bioconda to mulled automatic
        mapping. Should be the method of choice if mulled resolution is required
        but the container execution is handled by a container orchestration
@@ -28,7 +28,7 @@
        a tool is run.
        -->
 
-  <mulled_singularity auto_install="True"/>
+  <mulled_singularity auto_install="False"/>
   <!-- mulled_singularity: only resolves container URI through bioconda to mulled automatic
        mapping. Should be the method of choice if mulled resolution is required
        but the container execution is handled by a container orchestration
@@ -39,15 +39,15 @@
        a tool is run.
        -->
 
-  <!-- <build_mulled auto_install="True"/> -->
+  <!-- <build_mulled auto_install="False"/> -->
   <!-- build_mulled: builds a docker image locally for one or more bioconda
        packages listed in the tools definition.
        Set auto_install to False if Galaxy should build container images
        through the admin interface or API, but not automatically when
        a tool is run.
        -->
 
-  <!-- <build_mulled_singularity auto_install="True"/> -->
+  <!-- <build_mulled_singularity auto_install="False"/> -->
   <!-- build_mulled_singularity: builds a singularity image locally for one
        or more package requirements listed as part of the tool's definition.
        Set auto_install to False if Galaxy should build container images

diff --git a/compose/galaxy-configurator/templates/galaxy/dependency_resolvers_conf.xml b/compose/galaxy-configurator/templates/galaxy/dependency_resolvers_conf.xml
@@ -0,0 +1,3 @@
+<!-- TODO: make this conditional on using Singularity -->
+<dependency_resolvers>
+</dependency_resolvers>
diff --git a/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 b/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2
@@ -36,7 +36,7 @@
               <env id="HOME">/home/galaxy</env>
               <param id="singularity_enabled">true</param>
               {% if GALAXY_JOB_RUNNER == 'local' -%}
-                <param id="singularity_volumes">{{ EXPORT_DIR | regex_replace("^.", "") }}/$galaxy_root:$galaxy_root:ro,{{ EXPORT_DIR | regex_replace("^.", "") }}/$tool_directory:$tool_directory:ro,{{ EXPORT_DIR | regex_replace("^.", "") }}/$job_directory:$job_directory:rw,{{ EXPORT_DIR | regex_replace("^.", "") }}/$working_directory:$working_directory:rw,{{ EXPORT_DIR | regex_replace("^.", "") }}/$default_file_path:$default_file_path:rw</param>
+                <param id="singularity_volumes">{{ EXPORT_DIR | regex_replace("^.", "") }}/$galaxy_root:$galaxy_root:ro,{{ EXPORT_DIR | regex_replace("^.", "") }}/$tool_directory:$tool_directory:ro,{{ EXPORT_DIR | regex_replace("^.", "") }}/$job_directory:$job_directory:rw,{{ EXPORT_DIR | regex_replace("^.", "") }}/$working_directory:$working_directory:rw{%if GALAXY_SINGULARITY_VOLUMES is defined-%},{{GALAXY_SINGULARITY_VOLUMES}}{%endif-%},{{ EXPORT_DIR | regex_replace("^.", "") }}/$default_file_path:$default_file_path:rw</param>
               {% endif -%}
             {% elif GALAXY_DEPENDENCY_RESOLUTION == 'docker' -%}
               <param id="docker_enabled">true</param>

diff --git a/compose/galaxy-configurator/templates/galaxy/tool_sheds_conf.xml.j2 b/compose/galaxy-configurator/templates/galaxy/tool_sheds_conf.xml.j2
@@ -0,0 +1,7 @@
+<?xml version="1.0"?>
+<tool_sheds>
+    <tool_shed name="Galaxy Main Tool Shed" url="https://toolshed.g2.bx.psu.edu/"/>
+    {% if GALAXY_ENABLE_TESTTOOLSHED %}
+    <tool_shed name="Galaxy Test Tool Shed" url="https://testtoolshed.g2.bx.psu.edu/"/> 
+    {% endif %}
+</tool_sheds>