Merge branch 'release/v21.4.16'

portal/config/config.py

@@ -103,6 +103,7 @@ class BaseConfig(object):
         ('assessment_cache_region', 60*60*2),
         ('reporting_cache_region', 60*60*12)]
     SEND_FILE_MAX_AGE_DEFAULT = 60 * 60  # 1 hour, in seconds
+    ENABLE_2FA = os.environ.get('ENABLE_2FA', None)
 
     LOG_CACHE_MISS = False
     LOG_FOLDER = os.environ.get('LOG_FOLDER')

portal/config/eproms/Organization.json

@@ -3146,7 +3146,8 @@
         },
         {
           "research_protocols": [
-			{"name": "IRONMAN v3"}
+            {"name": "IRONMAN v3", "retired_as_of": "2021-02-04T03:00:00Z"},
+            {"name": "IRONMAN v5"}
           ],
           "url": "http://us.truenth.org/identity-codes/research-protocol"
         }
@@ -3174,7 +3175,8 @@
         },
         {
           "research_protocols": [
-			{"name": "IRONMAN v3"}
+            {"name": "IRONMAN v3", "retired_as_of": "2021-03-11T03:00:00Z"},
+            {"name": "IRONMAN v5"}
           ],
           "url": "http://us.truenth.org/identity-codes/research-protocol"
         }
@@ -3202,7 +3204,8 @@
         },
         {
           "research_protocols": [
-			{"name": "IRONMAN v3"}
+            {"name": "IRONMAN v3", "retired_as_of": "2021-03-27T03:00:00Z"},
+            {"name": "IRONMAN v5"}
           ],
           "url": "http://us.truenth.org/identity-codes/research-protocol"
         }

portal/factories/app.py

@@ -1,6 +1,7 @@
 """Portal module"""
 
 import email.parser
+import json_logging
 import logging
 from logging import handlers
 import os
@@ -244,6 +245,12 @@ def configure_logging(app):  # pragma: no cover
     if len(app.logger.handlers) > 1:
         return
 
+    # Configure for JSON logging
+    if json_logging._current_framework is None:
+        # Ugly internal ref to prevent multiple calls to `init_flask`
+        json_logging.init_flask(enable_json=True)
+        json_logging.init_request_instrument(app)
+
     if app.config.get('LOG_SQL'):
         import portal.sql_logging
 

portal/factories/celery.py

@@ -1,10 +1,26 @@
-from celery import Celery
+from celery import current_app, Celery, signals
+import logging
+import sys
 
 from ..extensions import db
 
 __celery = None
 
 
+@signals.setup_logging.connect
+def on_setup_logging(**kwargs):
+    # prefer loglevel from flask app config (injected into celery app) over celery CLI option
+    log_level = current_app.conf.get('LOG_LEVEL') or kwargs.get('loglevel')
+
+    logger = logging.getLogger('celery')
+    logger.setLevel(log_level)
+    logger.addHandler(logging.StreamHandler(sys.stdout))
+    logger.propagate = True
+    logger = logging.getLogger('celery.app.trace')
+    logger.setLevel(log_level)
+    logger.propagate = True
+
+
 def create_celery(app):
     global __celery
     if __celery:

portal/migrations/versions/406484da81af_add_user_otp_secret_for_2fa.py

@@ -0,0 +1,26 @@
+"""add user.otp_secret for 2FA
+
+Revision ID: 406484da81af
+Revises: 1a84c56e6abc
+Create Date: 2021-04-06 15:21:36.057351
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '406484da81af'
+down_revision = '1a84c56e6abc'
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('users', sa.Column('otp_secret', sa.String(length=16)))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('users', 'otp_secret')
+    # ### end Alembic commands ###

portal/models/login.py

@@ -1,13 +1,17 @@
 """Module for common login hook"""
 
 from flask import current_app, session
+from flask_babel import gettext as _
 from flask_login import (
     current_user as flask_login_current_user,
     login_user as flask_user_login,
+    logout_user as flask_user_logout,
 )
 from flask_user import _call_or_get
 
+from ..database import db
 from .encounter import initiate_encounter
+from .role import ROLE
 
 
 def login_user(user, auth_method=None):
@@ -20,6 +24,40 @@ def login_user(user, auth_method=None):
     authentication method.
 
     """
+    from .message import EmailMessage  # prevent cycle
+    # beyond patients and care givers, 2FA is required.  confirm or initiate
+    if (
+            current_app.config.get("ENABLE_2FA") and
+            not current_app.testing and
+            user.has_role(
+                    ROLE.ADMIN.value,
+                    ROLE.ANALYST.value,
+                    ROLE.CLINICIAN.value,
+                    ROLE.PRIMARY_INVESTIGATOR.value,
+                    ROLE.CLINICIAN.value,
+                    ROLE.RESEARCHER.value,
+                    ROLE.STAFF.value,
+                    ROLE.STAFF_ADMIN.value,
+            ) and
+            session.get('2FA_verified') != '2FA verified'):
+        # log user back out, in case a flow already promoted them
+        flask_user_logout()
+
+        token = user.generate_otp()
+        session['user_needing_2fa'] = user.id
+        session['pending_auth_method'] = auth_method
+        current_app.logger.debug(f"OTP for {user.id}: {token}")
+        email = EmailMessage(
+            subject=_("TrueNTH Access Token"),
+            body=_("One Time Authentication Token: %s" % token),
+            recipients=user.email,
+            sender=current_app.config['MAIL_DEFAULT_SENDER'],
+            user_id=user.id)
+        email.send_message()
+        db.session.add(email)
+        db.session.commit()
+        return
+
     if not _call_or_get(flask_login_current_user.is_authenticated):
         flask_user_login(user)
 

portal/models/qb_status.py

@@ -559,6 +559,10 @@ def patient_research_study_status(patient, ignore_QB_status=False):
                 # Clear ready status when base has pending work
                 rs_status['ready'] = False
                 rs_status['errors'].append('Pending work in base study')
+            elif not patient.email_ready():
+                # Avoid errors from automated emails, that is, email required
+                rs_status['ready'] = False
+                rs_status['errors'].append('User lacks valid email address')
             elif rs_status['ready']:
                 # As user may have just entered ready status on EMPRO
                 # move trigger_states.state to due

portal/models/qbd.py

@@ -94,6 +94,8 @@ def completed_date(self, user_id):
 
         """
         from .qb_timeline import QBT
+        from .questionnaire_bank import QuestionnaireBank
+        from .questionnaire_response import QuestionnaireResponse
         query = QBT.query.filter(QBT.user_id == user_id).filter(
             QBT.qb_id == self.qb_id).filter(
             QBT.qb_recur_id == self.recur_id).filter(
@@ -103,5 +105,19 @@ def completed_date(self, user_id):
             raise ValueError(
                 f"Should never find multiple completed for {user_id} {self}")
         if not query.count():
+            # Check indefinite case, which doesn't generate timeline rows
+            if self._questionnaire_bank.classification == 'indefinite':
+                found = QuestionnaireResponse.query.filter(
+                    QuestionnaireResponse.subject_id == user_id).filter(
+                    QuestionnaireResponse.status == 'completed').join(
+                    QuestionnaireBank).filter(
+                    QuestionnaireResponse.questionnaire_bank_id ==
+                    QuestionnaireBank.id).filter(
+                    QuestionnaireBank.classification ==
+                    'indefinite').with_entities(
+                    QuestionnaireResponse.document['authored'].label(
+                        'authored')).first()
+                if found:
+                    return FHIR_datetime.parse(found[0])
             return None
         return query.first().at

portal/models/reporting.py

@@ -10,7 +10,7 @@
 from ..audit import auditable_event
 from ..cache import cache
 from ..date_tools import FHIR_datetime
-from ..trigger_states.models import TriggerState, TriggerStatesReporting
+from ..trigger_states.models import TriggerStatesReporting
 from .app_text import MailResource, SiteSummaryEmail_ATMA, app_text
 from .communication import load_template_args
 from .message import EmailMessage
@@ -28,7 +28,7 @@
 from .research_study import EMPRO_RS_ID, ResearchStudy
 from .role import ROLE, Role
 from .user import User, UserRoles, patients_query
-from .user_consent import latest_consent
+from .user_consent import consent_withdrawal_dates
 
 
 def adherence_report(
@@ -94,10 +94,10 @@ def adherence_report(
             'site': patient.organizations[0].name,
             'status': str(qb_stats.overall_status)}
 
-        consent = latest_consent(
-            user=patient, research_study_id=research_study_id)
-        if consent:
-            row['consent'] = FHIR_datetime.as_fhir(consent.acceptance_date)
+        c_date, w_date = consent_withdrawal_dates(
+                user=patient, research_study_id=research_study_id)
+        consent = c_date if c_date else w_date
+        row['consent'] = FHIR_datetime.as_fhir(consent)
 
         study_id = patient.external_study_id
         if study_id:
@@ -109,6 +109,9 @@ def adherence_report(
         if last_viable:
             row['qb'] = last_viable.questionnaire_bank.name
             row['visit'] = visit_name(last_viable)
+            if row['status'] == 'Completed':
+                row['completion_date'] = FHIR_datetime.as_fhir(
+                    last_viable.completed_date(patient.id))
             entry_method = QNR_results(
                 patient,
                 research_study_id=research_study_id,
@@ -126,10 +129,10 @@ def adherence_report(
                     row['clinician'] = ';'.join(
                         clinician.display_name for clinician in
                         patient.clinicians)
-                # As we may be looking at `prev_qbd` can't use qb_stats
-                cd = last_viable.completed_date(patient.id)
-                if cd:
-                    row['EMPRO_questionnaire_completion_date'] = cd
+                # Rename column header for EMPRO
+                if 'completion_date' in row:
+                    row['EMPRO_questionnaire_completion_date'] = (
+                        row.pop('completion_date'))
 
                 # Correct for zero index visit month in db
                 visit_month = int(row['visit'].split()[-1]) - 1
@@ -152,6 +155,9 @@ def adherence_report(
             historic['status'] = status
             historic['qb'] = qbd.questionnaire_bank.name
             historic['visit'] = visit_name(qbd)
+            historic['completion_date'] = (
+                FHIR_datetime.as_fhir(qbd.completed_date(patient.id))
+                if status == 'Completed' else '')
             entry_method = QNR_results(
                 patient,
                 research_study_id=research_study_id,
@@ -176,16 +182,22 @@ def adherence_report(
                 historic['content_domains_accessed'] = (
                     ', '.join(da) if da else '')
 
-                cd = qbd.completed_date(patient.id)
-                if cd:
-                    historic['EMPRO_questionnaire_completion_date'] = cd
+                # Rename column header for EMPRO
+                if 'completion_date' in historic:
+                    historic['EMPRO_questionnaire_completion_date'] = (
+                        historic.pop('completion_date'))
             data.append(historic)
 
         # if user is eligible for indefinite QB, add status
         qbd, status = qb_stats.indef_status()
         if qbd:
             indef = row.copy()
             indef['status'] = status
+            # Indefinite doesn't have a row in the timeline, look
+            # up matching date from QNRs
+            indef['completion_date'] = (
+                FHIR_datetime.as_fhir(qbd.completed_date(patient.id))
+                if status == 'Completed' else '')
             indef['qb'] = qbd.questionnaire_bank.name
             indef['visit'] = "Indefinite"
             entry_method = QNR_results(
@@ -213,7 +225,7 @@ def adherence_report(
         results['filename_prefix'] = base_name
         results['column_headers'] = [
             'user_id', 'study_id', 'status', 'visit', 'entry_method', 'site',
-            'consent']
+            'consent', 'completion_date']
         if research_study_id == EMPRO_RS_ID:
             results['column_headers'] = [
                 'user_id',

portal/models/user.py

@@ -1,10 +1,12 @@
 """User model """
 
-
+import base64
 from html import escape
 from datetime import datetime, timedelta
 from io import StringIO
+import os
 import re
+import onetimepass as otp
 import time
 
 from dateutil import parser
@@ -256,6 +258,11 @@ def validate_email(email):
         raise BadRequest("requires a valid email address")
 
 
+def generate_random_secret():
+    """generate a random secret"""
+    return base64.b32encode(os.urandom(10)).decode('utf-8')
+
+
 class User(db.Model, UserMixin):
     # PLEASE maintain merge_with() as user model changes #
     __tablename__ = 'users'  # Override default 'user'
@@ -304,6 +311,9 @@ class User(db.Model, UserMixin):
         db.Column(db.Integer, default=0, nullable=False)
     last_password_verification_failure = db.Column(db.DateTime, nullable=True)
 
+    # For 2FA
+    otp_secret = db.Column(db.String(16), default=generate_random_secret)
+
     user_audits = db.relationship('Audit', cascade='delete',
                                   foreign_keys=[Audit.user_id])
     subject_audits = db.relationship('Audit', cascade='delete',
@@ -469,6 +479,28 @@ def current_encounter(
             return initiate_encounter(self, auth_method=existing.auth_method)
         return existing
 
+    TOTP_TOKEN_LEN = 4
+    TOTP_TOKEN_LIFE = 30*60
+
+    def generate_otp(self):
+        """Generate One Time Password for 2FA from user's otp_secret"""
+        if self.otp_secret is None:
+            self.otp_secret = generate_random_secret()
+            db.session.commit()
+
+        return otp.get_totp(
+            self.otp_secret,
+            token_length=self.TOTP_TOKEN_LEN,
+            interval_length=self.TOTP_TOKEN_LIFE)
+
+    def validate_otp(self, token):
+        assert(self.otp_secret)
+        return otp.valid_totp(
+            token,
+            self.otp_secret,
+            token_length=self.TOTP_TOKEN_LEN,
+            interval_length=self.TOTP_TOKEN_LIFE)
+
     @property
     def locale(self):
         if self._locale and self._locale.codings and (

portal/templates/flask_user/second_factor.html

@@ -0,0 +1,16 @@
+{% extends "layout.html" %}
+{% block main %}
+{% from "flask_user/_macros.html" import render_field %}
+
+    <h3 class="tnth-headline">{{ _("Two Factor Athentication") }}</h3><br />
+
+    <p>{{ _("You should receive an email with a token. Enter the token below for access.") }}</p>
+
+    <form action="{{ url_for('auth.two_factor_auth') }}" method="post">
+        {{ form.hidden_tag() }}
+        {{ render_field(form.key, label=_("Token"), tabindex=5) }}
+
+        <button type="submit" class="btn btn-tnth-primary">{{ _("Validate Token") }}</button>
+    </form>
+
+{% endblock %}