fix(reporting): Updated Sentry reporting; implemented automated versi…

…oning for CI routines
hms-dbmi · Jul 31, 2024 · 68f45be · 68f45be
1 parent e4e8a23
commit 68f45be
Show file tree

Hide file tree

Showing 16 changed files with 6,452 additions and 159 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -0,0 +1,60 @@
+name: Release Build
+
+on:
+  push:
+    branches: [ master, development ]
+  workflow_dispatch:
+    inputs:
+      force:
+        type: boolean
+        default: false
+
+jobs:
+  test-image:
+    uses: hms-dbmi/actions/.github/workflows/test-image-build.yml@main
+    secrets:
+      DOCKER_HUB_USERNAME: ${{ secrets.BLHMSDBMI_DOCKERHUB_USERNAME }}
+      DOCKER_HUB_PASSWORD: ${{ secrets.BLHMSDBMI_DOCKERHUB_PASSWORD }}
+    with:
+      repository: ${{ github.repository }}
+      commit: ${{ github.sha }}
+
+  scan-image:
+    uses: hms-dbmi/actions/.github/workflows/scan.yml@main
+    secrets:
+      DOCKER_HUB_USERNAME: ${{ secrets.BLHMSDBMI_DOCKERHUB_USERNAME }}
+      DOCKER_HUB_PASSWORD: ${{ secrets.BLHMSDBMI_DOCKERHUB_PASSWORD }}
+    with:
+      repository: ${{ github.repository }}
+      commit: ${{ github.sha }}
+
+  metadata:
+    runs-on: "ubuntu-latest"
+    outputs:
+      branch: ${{ steps.set_branch.outputs.branch }}
+      force: ${{ steps.force.outputs.force }}
+    steps:
+      - name: Set the current branch name
+        shell: bash
+        run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
+        id: set_branch
+      - name: Manage force input
+        id: force
+        env:
+          FORCE_INPUT: ${{ inputs.force }}
+        run: echo "force=${FORCE_INPUT:=false}" >> $GITHUB_OUTPUT
+
+  deploy:
+    uses: hms-dbmi/actions/.github/workflows/dbmisvc-app-deploy.yml@main
+    needs:
+      - "test-image"
+      - "metadata"
+    secrets:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    with:
+      filename: "${{ github.event.repository.name }}-${{ needs.metadata.outputs.branch }}.zip"
+      app: ${{ github.event.repository.name }}
+      role: ${{ vars[ format('BLHMSDBMI_DBMISVC_DEPLOYMENT_{0}_ROLE_ARN', needs.metadata.outputs.branch) ] }}
+      bucket: ${{ vars[ format('BLHMSDBMI_DBMISVC_DEPLOYMENT_{0}_BUCKET', needs.metadata.outputs.branch) ] }}
+      force: ${{ fromJson(needs.metadata.outputs.force) }}
+      backmerge: development
diff --git a/.github/workflows/scan.yml b/.github/workflows/scan.yml
@@ -1,8 +1,6 @@
 name: Scan
 
 on:
-  push:
-    branches: [ master, development ]
   pull_request:
     branches: [ master, development ]
   schedule:

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -1,8 +1,6 @@
 name: Test Image Build
 
 on:
-  push:
-    branches: [ master, development ]
   pull_request:
     branches: [ master, development ]
   workflow_dispatch:

diff --git a/.gitignore b/.gitignore
@@ -297,6 +297,8 @@ $RECYCLE.BIN/
 # Windows shortcuts
 *.lnk
 
+# Node modules
+node_modules
 
 # End of https://www.gitignore.io/api/osx,linux,django,python,windows,pycharm+all,visualstudiocode
 .vscode/settings.json
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,16 +1,21 @@
 minimum_pre_commit_version: "2.13.0"
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks.git
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
     - id: trailing-whitespace
     - id: mixed-line-ending
     - id: check-byte-order-marker
     - id: check-merge-conflict
     - id: detect-aws-credentials
+      args: [--allow-missing-credentials]
   - repo: https://github.com/jazzband/pip-tools
     rev: 7.4.1
     hooks:
+      - id: pip-compile
+        name: pip-compile test-requirements.in
+        args: [test-requirements.in, --upgrade, --generate-hashes, --allow-unsafe, --output-file, test-requirements.txt]
+        files: ^test-requirements\.(in|txt)$
       - id: pip-compile
         name: pip-compile dev-requirements.in
         args: [dev-requirements.in, --upgrade, --generate-hashes, --allow-unsafe, --output-file, dev-requirements.txt]

diff --git a/.releaserc b/.releaserc
@@ -0,0 +1,29 @@
+{
+    "branches": [
+        "master",
+        "main",
+        "+([0-9])?(.{+([0-9]),x}).x",
+        {"name": "development", "channel": "pre/rc", "prerelease": "rc"},
+        {"name": "beta", "prerelease": true},
+        {"name": "alpha", "prerelease": true}
+    ],
+    "plugins": [
+        "@semantic-release/commit-analyzer",
+        "@semantic-release/release-notes-generator",
+        ["@semantic-release/changelog", {
+            "changelogFile": "CHANGELOG.md"
+        }],
+        ["@semantic-release/exec", {
+          "prepareCmd": "sed -ie \"s/^version.*/version = \\x22${nextRelease.version}\\x22/g\" pyproject.toml"
+        }],
+        ["@semantic-release/git", {
+            "assets": ["pyproject.toml", "CHANGELOG.md"]
+        }],
+        ["@semantic-release/github", {
+            "successComment": false,
+            "failComment": false,
+            "failTitle": false
+        }]
+    ],
+    "preset": "angular"
+}
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM hmsdbmitc/dbmisvc:debian12-slim-python3.11-0.6.1 AS builder
+FROM hmsdbmitc/dbmisvc:debian12-slim-python3.11-0.6.2 AS builder
 
 # Install requirements
 RUN apt-get update \
@@ -20,7 +20,23 @@ RUN pip install -U wheel \
     && pip wheel -r /requirements.txt \
         --wheel-dir=/root/wheels
 
-FROM hmsdbmitc/dbmisvc:debian12-slim-python3.11-0.6.1
+FROM hmsdbmitc/dbmisvc:debian12-slim-python3.11-0.6.2
+
+ARG APP_NAME="dbmi-fileservice"
+ARG APP_CODENAME="dbmi-fileservice"
+ARG VERSION
+ARG COMMIT
+ARG DATE
+
+LABEL org.label-schema.schema-version=1.0 \
+    org.label-schema.vendor="HMS-DBMI" \
+    org.label-schema.version=${VERSION} \
+    org.label-schema.name=${APP_NAME} \
+    org.label-schema.build-date=${DATE} \
+    org.label-schema.description="DBMI Fileservice" \
+    org.label-schema.url="https://github.com/hms-dbmi/dbmi-fileservice" \
+    org.label-schema.vcs-url="https://github.com/hms-dbmi/dbmi-fileservice" \
+    org.label-schema.vcf-ref=${COMMIT}
 
 # Copy Python wheels from builder
 COPY --from=builder /root/wheels /root/wheels
@@ -50,6 +66,10 @@ ADD docker-entrypoint-init.d/* /docker-entrypoint-init.d/
 COPY /fileservice /app
 
 # Set app parameters. These can be overridden in the ECS Task Definition's container environment variables.
+ENV DBMI_APP_NAME=${APP_NAME}
+ENV DBMI_APP_CODENAME=${APP_CODENAME}
+ENV DBMI_APP_VERSION=${VERSION}
+ENV DBMI_APP_COMMIT=${COMMIT}
 ENV DBMI_ENV=prod
 ENV DBMI_PARAMETER_STORE_PREFIX=dbmi.fileservice.${DBMI_ENV}
 ENV DBMI_PARAMETER_STORE_PRIORITY=true

diff --git a/buildspec.yml b/buildspec.yml
@@ -4,22 +4,20 @@ env:
   parameter-store:
     DOCKER_HUB_USERNAME: "/dbmi/codebuild/docker/username"
     DOCKER_HUB_PASSWORD: "/dbmi/codebuild/docker/password"
-
 phases:
+  install:
+    commands:
+      - pip3 install toml-cli
   pre_build:
     commands:
       - echo Logging in to Amazon ECR...
       - $(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION)
       - echo Logging in to Docker Hub...
       - docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD}
-
   build:
     commands:
-      - echo Building docker image...
-      - docker build -t $IMAGE_REPO_NAME:$IMAGE_TAG .
-      - echo Tagging new docker image...
+      - docker build --build-arg "VERSION=`toml get project.version --toml-path pyproject.toml`" --build-arg "COMMIT=`cat ./COMMIT`" --build-arg "DATE=`date +'%Y-%m-%dT%H:%M:%S%z'`" -t $IMAGE_REPO_NAME:$IMAGE_TAG .
       - docker tag $IMAGE_REPO_NAME:$IMAGE_TAG $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$IMAGE_REPO_NAME:$IMAGE_TAG
-
   post_build:
     commands:
       - echo Build completed on `date`

diff --git a/fileservice/fileservice/settings.py b/fileservice/fileservice/settings.py
@@ -8,6 +8,8 @@
 import logging
 
 from dbmi_client import environment
+from dbmi_client import reporting
+from dbmi_client.logging import config
 
 # PATH CONFIGURATION
 
@@ -372,77 +374,49 @@
 
 AXES_ENABLED = environment.get_bool('DJANGO_AXES_ENABLED', True)
 
-# LOGGING CONFIGURATION
+#####################################################################################
+# Logging settings
+#####################################################################################
+
+# Configure  Sentry
+reporting.sentry(
+    sentry_dsn=environment.get_str("SENTRY_DSN", required=True),
+    release=f'{environment.get_str("DBMI_APP_NAME")}@{environment.get_str("DBMI_APP_VERSION")}',
+    environment=environment.get_str("DBMI_ENV", "prod"),
+    sentry_trace_rate=environment.get_float("SENTRY_TRACES_RATE", default=0.0),
+    sentry_profile_rate=environment.get_float("SENTRY_PROFILES_RATE", default=0.0),
+)
 
-# Configure sentry
-if environment.get_str('RAVEN_URL', default=None):
-    RAVEN_CONFIG = {
-        'dsn': environment.get_str('RAVEN_URL'),
-        'site': 'fileservice',
-    }
+# Output the standard logging configuration
+LOGGING = config('DBMI-FILESERVICE', root_level=logging.DEBUG)
 
-LOGGING = {
-    'version': 1,
-    'disable_existing_loggers': False,
-    'formatters': {
-        'console': {
-            'format': '[DBMI-Fileservice] - [%(asctime)s][%(levelname)s]'
-                      '[%(name)s.%(funcName)s:%(lineno)d] - %(message)s',
-        },
+# Disable warning level for 4xx request logging
+LOGGING['loggers'].update({
+    'django.request': {
+        'handlers': ['console'],
+        'level': 'ERROR',
+        'propagate': True,
     },
-    'handlers': {
-        'sentry': {
-            'level': 'ERROR',  # To capture more than ERROR, change to WARNING, INFO, etc.
-            'class': 'raven.contrib.django.raven_compat.handlers.SentryHandler',
-            'tags': {'custom-tag': 'x'},
-        },
-        'console': {
-            'level': 'DEBUG',
-            'class': 'logging.StreamHandler',
-            'formatter': 'console',
-            'stream': stdout,
-        }
+    'boto3': {
+        'handlers': ['console'],
+        'level': 'INFO',
+        'propagate': True,
     },
-    'root': {
-        'handlers': ['console', 'sentry', ],
-        'level': 'DEBUG',
+    'botocore': {
+        'handlers': ['console'],
+        'level': 'INFO',
+        'propagate': True,
     },
-    'loggers': {
-        'django': {
-            'handlers': ['console', ],
-            'level': 'ERROR',
-            'propagate': True,
-        },
-        'raven': {
-            'level': 'WARNING',
-            'handlers': ['console'],
-            'propagate': False,
-        },
-        'sentry.errors': {
-            'level': 'WARNING',
-            'handlers': ['console'],
-            'propagate': False,
-        },
-        'botocore': {
-            'level': 'WARNING',
-            'handlers': ['console'],
-            'propagate': True,
-        },
-        'boto': {
-            'level': 'WARNING',
-            'handlers': ['console'],
-            'propagate': True,
-        },
-        'boto3': {
-            'level': 'WARNING',
-            'handlers': ['console'],
-            'propagate': True,
-        },
-        's3transfer': {
-            'level': 'WARNING',
-            'handlers': ['console'],
-            'propagate': True,
-        },
+    's3transfer': {
+        'handlers': ['console'],
+        'level': 'INFO',
+        'propagate': True,
     },
-}
-# END LOGGING CONFIGURATION
+    'urllib3': {
+        'handlers': ['console'],
+        'level': 'INFO',
+        'propagate': True,
+    },
+})
+
+#####################################################################################