#1858 Kibana access + changes for fully local development

tdrs-backend/.env.example

@@ -2,6 +2,8 @@
 # Copy this file to `.env` and replace variables as needed
 # 
 
+#REACT_APP_DEVAUTH=1
+
 # ##
 # Required environment variables
 # These must be defined or the application will encounter fatal errors

tdrs-backend/Pipfile

@@ -58,7 +58,7 @@ django-elasticsearch-dsl = "==7.3"
 django-elasticsearch-dsl-drf = "==0.22.5"
 requests-aws4auth = "==1.1.2"
 cerberus = "==1.3.4"
-xlsxwriter = "==3.0.1"
+xlsxwriter = "==3.1.9"
 sendgrid = "==6.10.0"
 
 [requires]

tdrs-backend/docker-compose.yml

@@ -45,12 +45,68 @@ services:
       # Copy in the Localstack setup script to configure any buckets needed
       - ../scripts/localstack-setup.sh:/docker-entrypoint-initaws.d/localstack-setup.sh
 
+  kibana:
+    image: elastic/kibana:7.17.10
+    ports:
+      - 5601:5601
+    environment:
+      - xpack.security.encryptionKey="something_at_least_32_characters"
+      - xpack.security.session.idleTimeout="1h"
+      - xpack.security.session.lifespan="30d"
+    volumes:
+      - ./kibana.yml:/usr/share/kibana/config/kibana.yml
+    depends_on:
+      - elastic
+
+  # This task only needs to be performed once, during the *initial* startup of
+  # the stack. Any subsequent run will reset the passwords of existing users to
+  # the values defined inside the '.env' file, and the built-in roles to their
+  # default permissions.
+  #
+  # By default, it is excluded from the services started by 'docker compose up'
+  # due to the non-default profile it belongs to. To run it, either provide the
+  # '--profile=elastic_setup' CLI flag to Compose commands, or "up" the service by name
+  # such as 'docker compose up elastic_setup'.
+  elastic_setup:
+    profiles:
+      - elastic_setup
+    build:
+      context: elastic_setup/
+      args:
+        ELASTIC_VERSION: "7.17.6"
+    init: true
+    environment:
+      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}
+      KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-}
+      DATA_ANALYST_PASSWORD: ${DATA_ANALYST_PASSWORD:-}
+      OFA_SYS_ADMIN_PASSWORD: ${OFA_SYS_ADMIN_PASSWORD:-}
+      ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:-}
+    depends_on:
+      - elastic
+    network_mode: "host"
+
   elastic:
     image: elasticsearch:7.17.6
     environment:
       - discovery.type=single-node
-      - xpack.security.enabled=false
       - logger.discovery.level=debug
+      - xpack.security.enabled=true
+      # - xpack.security.authc.realms.oidc.hhsams # The name of our realm
+      # - xpack.security.authc.realms.oidc.hhsams.order=2 # Should be highest in the realm chain
+      # - xpack.security.authc.realms.oidc.hhsams.rp.client_id="the_client_id" # assigned to the Elastic Stack RP by the OP upon registration.
+      # - xpack.security.authc.realms.oidc.hhsams.rp.response_type=code # controls which OpenID Connect authentication flow this RP supports: code, token, id_token
+      # - xpack.security.authc.realms.oidc.hhsams.rp.redirect_uri="https://kibana.example.org:5601/api/security/oidc/callback"
+      # - xpack.security.authc.realms.oidc.hhsams.op.issuer="https://op.example.org" # Issuer Identifier is usually a case sensitive UR
+      # - xpack.security.authc.realms.oidc.hhsams.op.authorization_endpoint="https://op.example.org/oauth2/v1/authorize" # The URL for the Authorization Endpoint in the OP
+      # - xpack.security.authc.realms.oidc.hhsams.op.token_endpoint="https://op.example.org/oauth2/v1/token" # The URL for the Token Endpoint in the OpenID Connect Provider
+      # - xpack.security.authc.realms.oidc.hhsams.op.jwkset_path=oidc/jwkset.json # The path to a file or a URL containing a JSON Web Key Set with the key material that the OpenID Connect Provider uses for signing tokens and claims responses. OP should provide this.
+      # - xpack.security.authc.realms.oidc.hhsams.op.userinfo_endpoint="https://op.example.org/oauth2/v1/userinfo" # (Optional) The URL for the UserInfo Endpoint in the OpenID Connect Provider.
+      # - xpack.security.authc.realms.oidc.hhsams.op.endsession_endpoint="https://op.example.org/oauth2/v1/logout" # (Optional) The URL to the End Session Endpoint in the OpenID Connect Provider.
+      # - xpack.security.authc.realms.oidc.hhsams.rp.post_logout_redirect_uri="https://kibana.example.org:5601/security/logged_out" # (Optional) The Redirect URL where the OpenID Connect Provider should redirect the user after a successful Single Logout (assuming op.endsession_endpoint above is also set).
+      # - xpack.security.authc.realms.oidc.hhsams.claims.principal=sub
+      # - xpack.security.authc.realms.oidc.hhsams.claims.groups="http://example.info/claims/groups"
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD:-}
+      - KIBANA_SYSTEM_PASSWORD=${KIBANA_SYSTEM_PASSWORD:-}
     ports:
       - 9200:9200
       - 9300:9300

tdrs-backend/elastic_setup/Dockerfile

@@ -0,0 +1,10 @@
+ARG ELASTIC_VERSION
+
+FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION}
+
+COPY . /
+
+RUN ["chmod", "+x", "/entrypoint.sh"]
+RUN ["chmod", "+x", "/util.sh"]
+
+ENTRYPOINT ["/entrypoint.sh"]

tdrs-backend/elastic_setup/entrypoint.sh

@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+source "${BASH_SOURCE[0]%/*}"/util.sh
+
+
+# --------------------------------------------------------
+# Users declarations
+
+declare -A users_passwords
+users_passwords=(
+	[kibana_system]="${KIBANA_SYSTEM_PASSWORD:-}"
+	[data_analyst]="${DATA_ANALYST_PASSWORD:-}"
+	[ofa_sys_admin]="${OFA_SYS_ADMIN_PASSWORD:-}"
+)
+
+declare -A users_roles
+users_roles=(
+	[kibana_system]='kibana_system'
+    [data_analyst]='editor'
+    [ofa_sys_admin]='kibana_admin'
+)
+
+# --------------------------------------------------------
+# Roles declarations for custom roles
+
+declare -A roles_files
+roles_files=(
+
+)
+
+# --------------------------------------------------------
+
+
+log 'Waiting for availability of Elasticsearch. This can take several minutes.'
+
+declare -i exit_code=0
+wait_for_elasticsearch || exit_code=$?
+
+if ((exit_code)); then
+	case $exit_code in
+		6)
+			suberr 'Could not resolve host. Is Elasticsearch running?'
+			;;
+		7)
+			suberr 'Failed to connect to host. Is Elasticsearch healthy?'
+			;;
+		28)
+			suberr 'Timeout connecting to host. Is Elasticsearch healthy?'
+			;;
+		*)
+			suberr "Connection to Elasticsearch failed. Exit code: ${exit_code}"
+			;;
+	esac
+
+	exit $exit_code
+fi
+
+sublog 'Elasticsearch is running'
+
+log 'Waiting for initialization of built-in users'
+
+wait_for_builtin_users || exit_code=$?
+
+if ((exit_code)); then
+	suberr 'Timed out waiting for condition'
+	exit $exit_code
+fi
+
+sublog 'Built-in users were initialized'
+
+for role in "${!roles_files[@]}"; do
+	log "Role '$role'"
+
+	declare body_file
+	body_file="${BASH_SOURCE[0]%/*}/roles/${roles_files[$role]:-}"
+	if [[ ! -f "${body_file:-}" ]]; then
+		sublog "No role body found at '${body_file}', skipping"
+		continue
+	fi
+
+	sublog 'Creating/updating'
+	ensure_role "$role" "$(<"${body_file}")"
+done
+
+for user in "${!users_passwords[@]}"; do
+	log "User '$user'"
+	if [[ -z "${users_passwords[$user]:-}" ]]; then
+		sublog 'No password defined, skipping'
+		continue
+	fi
+
+	declare -i user_exists=0
+	user_exists="$(check_user_exists "$user")"
+
+	if ((user_exists)); then
+		sublog 'User exists, setting password'
+		set_user_password "$user" "${users_passwords[$user]}"
+	else
+		if [[ -z "${users_roles[$user]:-}" ]]; then
+			suberr '  No role defined, skipping creation'
+			continue
+		fi
+
+		sublog 'User does not exist, creating'
+		create_user "$user" "${users_passwords[$user]}" "${users_roles[$user]}"
+	fi
+done
+
+log "Elastic setup completed. Exiting with code: $?"