Add support for running IRIDA

compose/docker-compose.irida.yml

@@ -0,0 +1,61 @@
+# Extend Galaxy to use Singularity for dependency resolution.
+# This is working with the base Galaxy, but also in combination
+# with different job runners, like HTCondor, or Slurm
+# (Pulsar is still WIP).
+# Examples:
+#  * `docker-compose -f docker-compose.yml -f docker-compose.singularity.yml up`
+#  * `docker-compose -f docker-compose.yml -f docker-compose.slurm.yml -f docker-compose.singularity.yml up`
+version: "3.7"
+services:
+  galaxy-server:
+    environment:
+      - GALAXY_EXTRA_GROUP=root
+      - GALAXY_URL=http://nginx:90/
+    volumes:
+      - irida_web_data:/data/irida
+  galaxy-configurator:
+    environment:
+      - GALAXY_SINGULARITY_VOLUMES=/data/irida:rw
+      - GALAXY_CONFIG_ALLOW_PATH_PASTE=true
+      - IRIDA_OVERWRITE_CONFIG=true
+      - IRIDA_CONF_DIR=/etc/irida
+      - IRIDA_ENABLED=true
+    volumes:
+      - ${EXPORT_DIR:-./export}/irida:/etc/irida
+  nginx:
+    ports:
+      - 90:90
+  irida-web:
+    build: irida
+    container_name: irida-web
+    depends_on:
+      - irida-mysql
+    ports:
+      - "8080:8080"
+    volumes:
+      - ${EXPORT_DIR:-./export}/irida:/etc/irida
+      - irida_web_data:/data/irida
+    networks:
+      - galaxy
+
+  irida-mysql:
+    restart: always
+    image: mariadb:10.5.6
+    container_name: irida-mysql
+    environment:
+      MYSQL_ROOT_PASSWORD: rootpasswd
+      MYSQL_DATABASE: irida_db
+      MYSQL_USER: irida_user
+      MYSQL_PASSWORD: irida_passwd
+    volumes:
+      - irida_mysql_data:/var/lib/mysql
+    ports:
+      - "33060:3306"   
+    networks:
+      - galaxy
+
+volumes:
+  irida_mysql_data:
+  irida_web_data:
+  irida_galaxy_data:      
+

compose/docker-compose.yml

@@ -30,6 +30,7 @@ services:
     environment:
       - EXPORT_DIR=${EXPORT_DIR:-./export}
       - HOST_PWD=$PWD
+      - GALAXY_CONF_DIR=/galaxy/config
       - GALAXY_OVERWRITE_CONFIG=true
       - GALAXY_DEPENDENCY_RESOLUTION=conda
       - GALAXY_JOB_RUNNER=local
@@ -40,6 +41,7 @@ services:
       - GALAXY_CONFIG_AMQP_INTERNAL_CONNECTION=amqp://galaxy:vaiJa3ieghai2ief0jao@rabbitmq/galaxy
       - GALAXY_PROXY_PREFIX=${GALAXY_PROXY_PREFIX:-}
       - GALAXY_CONFIG_CLEANUP_JOB=onsuccess
+      - GALAXY_ENABLE_TESTTOOLSHED=true
       - NGINX_OVERWRITE_CONFIG=true
     volumes:
       - ${EXPORT_DIR:-./export}/galaxy/config:/galaxy/config
@@ -89,3 +91,6 @@ services:
       - galaxy
 networks:
   galaxy:
+    driver: bridge
+    driver_opts:
+      com.docker.network.driver.mtu: 1450

compose/extra-tools.yml

@@ -0,0 +1,7 @@
+tools:
+  - name: "data_manager_primer_scheme_bedfiles"
+    owner: "iuc"
+    tool_shed_url: "toolshed.g2.bx.psu.edu"
+    revisions:
+      - "157a5c9ea72a"
+    tool_panel_section_label: "COMBAT-SARS-COV-2::ARTIC-MINION"

compose/galaxy-configurator/Dockerfile

@@ -1,6 +1,6 @@
 FROM alpine:3.11
 
-RUN apk add --no-cache bash python3 \
+RUN apk add --no-cache bash python3 rsync \
     && pip3 install j2cli[yaml] jinja2-ansible-filters
 
 COPY ./templates /templates

compose/galaxy-configurator/run.sh

@@ -6,10 +6,11 @@ export GALAXY_CONF_DIR=${GALAXY_CONF_DIR:-/galaxy/config} \
        SLURM_CONF_DIR=${SLURM_CONF_DIR:-/etc/slurm-llnl} \
        HTCONDOR_CONF_DIR=${HTCONDOR_CONF_DIR:-/htcondor} \
        PULSAR_CONF_DIR=${PULSAR_CONF_DIR:-/pulsar/config} \
-       KIND_CONF_DIR=${KIND_CONF_DIR:-/kind}
+       KIND_CONF_DIR=${KIND_CONF_DIR:-/kind} \
+       IRIDA_CONF_DIR=${IRIDA_CONF_DIR:-/etc/irida}
 
 echo "Locking all configurations"
-locks=("$GALAXY_CONF_DIR" "$SLURM_CONF_DIR" "$HTCONDOR_CONF_DIR" "$PULSAR_CONF_DIR" "$KIND_CONF_DIR")
+locks=("$GALAXY_CONF_DIR" "$SLURM_CONF_DIR" "$HTCONDOR_CONF_DIR" "$PULSAR_CONF_DIR" "$KIND_CONF_DIR" "$IRIDA_CONF_DIR")
 for lock in "${locks[@]}"; do
   echo "Locking $lock"
   touch "${lock}/configurator.lock"
@@ -111,6 +112,25 @@ else
   chmod a+r "${GALAXY_KUBECONFIG:-${KIND_CONF_DIR}/.kube/config_in_docker}"
 fi
 
+# IRIDA configuration
+if [ "$IRIDA_OVERWRITE_CONFIG" != "true" ]; then
+  echo "IRIDA_OVERWRITE_CONFIG is not true. Skipping configuration of IRIDA"
+else
+  irida_configs=( "irida.conf" "web.conf" "static" "templates" "plugins" )
+
+  for conf in "${irida_configs[@]}"; do
+    echo "Configuring $conf"
+    if [[ "$conf" == *.conf ]] ; then
+      j2 --customize /customize.py --undefined -o "/tmp/$conf" "/templates/irida/$conf.j2" /base_config.yml
+      echo "The following changes will be applied to $conf:"
+      diff "${IRIDA_CONF_DIR}/$conf" "/tmp/$conf"
+      mv -f "/tmp/$conf" "${IRIDA_CONF_DIR}/$conf"
+    else
+      rsync -aP --delete "/templates/irida/$conf" "${IRIDA_CONF_DIR}/"
+    fi
+  done
+fi
+
 echo "Releasing all locks (except Galaxy) if it didn't happen already"
 locks=("$SLURM_CONF_DIR" "$HTCONDOR_CONF_DIR" "$PULSAR_CONF_DIR" "$KIND_CONF_DIR")
 for lock in "${locks[@]}"; do
@@ -138,7 +158,7 @@ if [ ! -f /base_config.yml ]; then
   touch /base_config.yml
 fi
 
-galaxy_configs=( "job_conf.xml" "galaxy.yml" "job_metrics.xml" "container_resolvers_conf.xml" "GALAXY_PROXY_PREFIX.txt" )
+galaxy_configs=( "tool_sheds_conf.xml" "job_conf.xml" "galaxy.yml" "job_metrics.xml" "container_resolvers_conf.xml" "GALAXY_PROXY_PREFIX.txt" )
 
 for conf in "${galaxy_configs[@]}"; do
   echo "Configuring $conf"

compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2

@@ -36,7 +36,7 @@
               <env id="HOME">/home/galaxy</env>
               <param id="singularity_enabled">true</param>
               {% if GALAXY_JOB_RUNNER == 'local' -%}
-                <param id="singularity_volumes">{{ EXPORT_DIR | regex_replace("^.", "") }}/$galaxy_root:$galaxy_root:ro,{{ EXPORT_DIR | regex_replace("^.", "") }}/$tool_directory:$tool_directory:ro,{{ EXPORT_DIR | regex_replace("^.", "") }}/$job_directory:$job_directory:rw,{{ EXPORT_DIR | regex_replace("^.", "") }}/$working_directory:$working_directory:rw,{{ EXPORT_DIR | regex_replace("^.", "") }}/$default_file_path:$default_file_path:rw</param>
+                <param id="singularity_volumes">{{ EXPORT_DIR | regex_replace("^.", "") }}/$galaxy_root:$galaxy_root:ro,{{ EXPORT_DIR | regex_replace("^.", "") }}/$tool_directory:$tool_directory:ro,{{ EXPORT_DIR | regex_replace("^.", "") }}/$job_directory:$job_directory:rw,{{ EXPORT_DIR | regex_replace("^.", "") }}/$working_directory:$working_directory:rw{%if GALAXY_SINGULARITY_VOLUMES is defined-%},{{GALAXY_SINGULARITY_VOLUMES}}{%endif-%},{{ EXPORT_DIR | regex_replace("^.", "") }}/$default_file_path:$default_file_path:rw</param>
               {% endif -%}
             {% elif GALAXY_DEPENDENCY_RESOLUTION == 'docker' -%}
               <param id="docker_enabled">true</param>

compose/galaxy-configurator/templates/galaxy/tool_sheds_conf.xml.j2

@@ -0,0 +1,7 @@
+<?xml version="1.0"?>
+<tool_sheds>
+    <tool_shed name="Galaxy Main Tool Shed" url="https://toolshed.g2.bx.psu.edu/"/>
+    {% if GALAXY_ENABLE_TESTTOOLSHED %}
+    <tool_shed name="Galaxy Test Tool Shed" url="https://testtoolshed.g2.bx.psu.edu/"/> 
+    {% endif %}
+</tool_sheds>

compose/galaxy-configurator/templates/irida/irida.conf.j2

@@ -0,0 +1,145 @@
+##### The filesystem location where files managed by irida are stored. The platform
+##### will *NOT* automatically make this directory, so it must exist before you
+##### start any instance of the platform.
+
+sequence.file.base.directory=/data/irida/sequence
+reference.file.base.directory=/data/irida/reference
+output.file.base.directory=/data/irida/output
+assembly.file.base.directory=/data/irida/assembly
+
+# production / master directory store 
+
+#sequence.file.base.directory=/cip0/software/galaxy/cbtgalaxydev/cbtgx/irida/data/irida/sequence
+#reference.file.base.directory=/cip0/software/galaxy/cbtgalaxydev/cbtgx/irida/data/irida/reference
+#output.file.base.directory=/cip0/software/galaxy/cbtgalaxydev/cbtgx/irida/data/irida/output
+
+##### Set the max upload size (in bytes). If left unconfigured, the max upload
+##### size is unlimited (or limited by the container hosting IRIDA).
+# file.upload.max_size=
+
+##### Set number of threads for FASTQC and file post-processsing.  The max size
+##### should not be more than the number of jdbc threads.
+file.processing.core.size=4
+file.processing.max.size=8
+file.processing.queue.capacity=512
+
+
+# don't decompress compressed data
+file.processing.decompress=false
+
+##### The database-specific settings. Several examples of how to specify a
+##### Hibernate driver are listed below (but commented out).
+
+## MySQL (or MariaDB)
+jdbc.driver=com.mysql.jdbc.Driver
+hibernate.dialect=org.hibernate.dialect.MySQL55Dialect
+
+## Database location (you may need to use a different string for different
+## database vendors).
+jdbc.url=jdbc:mysql://irida-mysql:3306/irida_db?allowPublicKeyRetrieval=true&useSSL=false
+
+## Connection settings:
+jdbc.username=irida_user
+jdbc.password=irida_passwd
+
+## Configuring Liquibase to execute a schema update. Should only make changes to
+## the database when executing the first time, or when upgrading.
+liquibase.update.database.schema=true
+
+## Configure Hibernate to execute a schema construction. WARNING: do not use this
+## at the same time as the Liquibase schema update. Liquibase will *not* execute
+## if this value is set, warnings will be produced in the log. These settings should
+## only be used in a development environment (**not** production).
+hibernate.hbm2ddl.auto=
+hibernate.hbm2ddl.import_files=
+
+## Configure Hibernate to show SQL in the log file. You *probably* don't want
+## to enable this, but could be useful for debugging.
+hibernate.show_sql=false
+
+## Connection Pool settings:
+jdbc.pool.initialSize=10
+jdbc.pool.maxActive=20
+jdbc.pool.testOnBorrow=true
+jdbc.pool.testOnReturn=true
+jdbc.pool.testWhileIdle=true
+
+## Configure the JDBC library to use this query to verify that a managed
+## connection is still valid. This may need to change, depending on your database vendor.
+jdbc.pool.validationQuery=select 1
+
+jdbc.pool.maxWait=10000
+jdbc.pool.removeAbandoned=true
+jdbc.pool.logAbandoned=true
+jdbc.pool.removeAbandonedTimeout=60
+jdbc.pool.maxIdle=10
+
+###############################################################################
+# Execution Manager configuration Galaxy. This is how IRIDA should connect to #
+# the internally managed instance of Galaxy for executing workflows.          #
+###############################################################################
+
+# The URL for the Galaxy execution manager
+# Networked service-to-service communication use the CONTAINER_PORT
+galaxy.execution.url=http://nginx:90/
+
+# The API key of an account to run workflows in Galaxy.
+# This does not have to be an administrator account.
+galaxy.execution.apiKey=fakekey
+
+# The email address of an account to run workflows in Galaxy
+galaxy.execution.email=admin@galaxy.org
+
+# The data storage method for uploading data into a Galaxy execution manager.
+galaxy.execution.dataStorage=local
+
+##################################
+# Workflow configuration options #
+##################################
+
+# The timeout (in seconds) for uploading files to Galaxy for execution
+# Increase this value if uploading files to Galaxy is timing out.
+galaxy.library.upload.timeout=10800
+
+# The polling time (in seconds) for checking if files have been uploaded to Galaxy
+# This value should not be greater than $galaxy.library.upload.timeout
+#galaxy.library.upload.polling.time=5
+
+# Number of threads used to wait for completion of uploading files.
+#galaxy.library.upload.threads=1
+
+# Maximum number of workflows IRIDA will schedule to run at the same time
+irida.workflow.max-running=4
+
+##################################
+# Analysis configuration options #
+##################################
+
+# The number days before intermediate files for an analysis get cleaned up.
+# That is, the number of days before files in Galaxy get deleted for the analysis.
+# Leave commented out for no cleanup.
+# This value can be fractional representing a fraction of a day (e.g. 0.5 for half a day).
+#irida.analysis.cleanup.days=
+
+#################################
+# Scheduled Task  configuration #
+#################################
+#Cron string for how often the email subscriptions are sent out.
+#Format: sec min hrs dom mon dow
+irida.scheduled.subscription.cron=0 0 0 * * *
+irida.scheduled.threads=2
+
+#################################
+# NCBI SRA Export configuration #
+#################################
+#Host to upload ncbi exports
+ncbi.upload.host=localhost
+#FTP Username and password for bulk SRA uploads
+ncbi.upload.user=test
+ncbi.upload.password=password
+#base directory in which to create SRA submissions
+ncbi.upload.baseDirectory=tmp
+#port for ftp upload
+ncbi.upload.port=21
+#Default namespace to preface file identifiers
+ncbi.upload.namespace=IRIDA

compose/galaxy-configurator/templates/irida/templates/login.html

@@ -0,0 +1,75 @@
+<!DOCTYPE html>
+<html xmlns:th="http://www.thymeleaf.org" th:lang="${#locale}">
+  <head>
+    <title th:text="#{LoginPage.title}">COMBAT-SARS-COV-2 Explorer</title>
+    <script th:inline="javascript">
+        window.TL = {
+            _BASE_URL: /*[[@{/}]]*/ "/",
+            emailConfigured: /*[[${emailConfigured}]]*/ false
+        };
+    </script>
+    <link rel="stylesheet" th:href="@{/static/css/bootstrap.min.css}" />
+    <link rel="stylesheet" th:href="@{/dist/css/login.bundle.css}" />
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+    <style>
+      body {
+        display: flex;
+        height: 100vh;
+        justify-content: center;
+        align-items: center;
+        flex-direction: column;
+      }
+      #login-root {
+        max-width: 400px;
+        padding: 10px;
+      }
+      .with-margin {
+         margin-bottom: 50px;
+      }
+    </style>
+  </head>
+  <body>
+    <div class="container">
+        <div class="row">
+            <div class="col-sm">
+                <div id="login-root">
+                    <!-- Contents rendered from LoginPage.jsx -->
+                </div>                            
+            </div>
+            <div class="col-sm">
+                <h1 class="h1"><img th:src="@{/static/logos/combat_tb_logo_small.png}"> COMBAT SARS-COV-2 Workbench</h1>
+                <p>The COMBAT-SARS-COV-2 Workbench provides easy to use tools for analysing <em>SARS-COV-2</em> genomic
+                    sequence samples.</p>
+                <p>It was developed at part of the <a href="https://combattb.org">COMBAT-SARS-COV-2 project</a>, an Africa CDC funded
+                    project undertaken by researchers and software developers at <a href="https://www.sanbi.ac.za">SANBI</a>. The COMBAT-SARS-COV-2
+                    Workbench is developed using technology from the <a href="https://irida.ca">IRIDA</a> and 
+                    <a href="https://galaxyproject.org">Galaxy</a> projects and incorporates leading tools developed for <em>SARS-COV-2</em>
+                    and microbial bioinformatics.</p>
+            </div>
+        </div>
+    </div>
+    <div class="container">
+        <div class="row justify-content-between with-margin">
+            <div class="col">
+                <img class="d-block m-auto" th:src="@{/static/logos/SANBI_logo_small.png}" alt="SANBI logo">
+            </div>
+            <div class="col">
+                <img class="d-block m-auto" th:src="@{/static/logos/UWC_logo_small.png}" alt="UWC logo">
+            </div>
+        </div>
+        <div class="row justify-content-between">
+            <div class="col">
+                <img th:src="@{/static/logos/SAMRCnewLogo.png}" alt="SA MRC logo">            
+            </div>
+            <div class="col">
+                <img th:src="@{/static/logos/dsi_logo_small.png}" alt="SA Department of Science and Innovation logo">
+            </div>
+            <div class="col">
+                <img th:src="@{/static/logos/NRF_logo_small.png}" alt="NRF logo">                        
+            </div>
+        </div>    
+    </div>
+    <!-- note: bootstrap-supporting javascript is not included as only a subset of bootstrap components are used -->
+    <script th:src="@{/dist/js/login.bundle.js}"></script>
+  </body>
+</html>