bring dev changes to master; prepare for release v24.12.12 (#4428)

portal/migrations/versions/4f5daa2b48db_.py

@@ -0,0 +1,24 @@
+"""remove non english rows from report data (IRONN-264)
+
+Revision ID: 4f5daa2b48db
+Revises: 5a300be640fb
+Create Date: 2024-11-25 13:48:01.321510
+
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = '4f5daa2b48db'
+down_revision = '5a300be640fb'
+
+
+def upgrade():
+    connection = op.get_bind()
+    connection.execute("DELETE FROM research_data WHERE NOT (data->>'timepoint' ~ '^(Baseline|Month)');")
+    # next run of `cache_research_data()` will pick up those just deleted.
+
+
+def downgrade():
+    # no point in bringing those back
+    pass

portal/models/qb_timeline.py

@@ -1316,8 +1316,8 @@ def qb_status_visit_name(user_id, research_study_id, as_of_date):
 
     :returns: dictionary with key/values for:
       status: string like 'expired'
-      visit_name: for the period, i.e. '3 months'
-      action_state: 'not applicable', or status of follow up action
+      visit_name: for the period, i.e. '3 months'. ALWAYS in english, clients must translate
+      action_state: 'not applicable', or status of follow-up action
 
     """
     from .research_study import EMPRO_RS_ID

portal/models/questionnaire_bank.py

@@ -592,6 +592,10 @@ def qbs_by_rp(rp_id, classification):
 
 
 def visit_name(qbd):
+    """returns string repr of visit, i.e. 'Month 3' or 'Baseline'
+
+    NB - only returns english version.  See `translate_visit_name()`
+    """
     from .research_study import (
         EMPRO_RS_ID,
         research_study_id_from_questionnaire,
@@ -617,12 +621,22 @@ def visit_name(qbd):
         clm += (clrd.years * 12) if clrd.years else 0
         total = clm * qbd.iteration + sm
         if rs_id == EMPRO_RS_ID:
-            return _('Month %(month_total)d', month_total=total+1)
-        return _('Month %(month_total)d', month_total=total)
+            return f'Month {total+1}'
+        return f'Month {total}'
 
     if rs_id == EMPRO_RS_ID:
-        return _('Month %(month_total)d', month_total=1)
-    return _(qbd.questionnaire_bank.classification.title())
+        return 'Month 1'
+    return qbd.questionnaire_bank.classification.title()
+
+
+def translate_visit_name(visit_name):
+    """parse the english version of visit name for front end translation needs"""
+    if not visit_name:
+        return visit_name
+    if visit_name.startswith('Month '):
+        number = int(visit_name[6:])
+        return _('Month %(month_total)d', month_total=number)
+    return _(visit_name)
 
 
 def add_static_questionnaire_bank():

portal/models/questionnaire_response.py

@@ -28,6 +28,7 @@
     QuestionnaireBank,
     QuestionnaireBankQuestionnaire,
     trigger_date,
+    translate_visit_name,
     visit_name,
 )
 from .research_data import ResearchData
@@ -401,7 +402,7 @@ def extensions(self):
                 relative_start=None, iteration=self.qb_iteration,
                 recur_id=recur_id, qb_id=self.questionnaire_bank_id)
             results.append({
-                'visit_name': visit_name(qbd),
+                'visit_name': translate_visit_name(visit_name(qbd)),
                 'url': TRUENTH_VISIT_NAME_EXTENSION})
 
             expires_at = expires(self.subject_id, qbd)
@@ -909,11 +910,16 @@ def qnr_document_id(
         QuestionnaireResponse.subject_id == subject_id).filter(
         QuestionnaireResponse.document[
             ('questionnaire', 'reference')
-        ].astext.endswith(questionnaire_name)).filter(
-        QuestionnaireResponse.questionnaire_bank_id ==
-        questionnaire_bank_id).with_entities(
+        ].astext.endswith(questionnaire_name)).with_entities(
         QuestionnaireResponse.document[(
             'identifier', 'value')])
+    if questionnaire_name != 'irondemog_v3':
+        # Another special indefinite workaround. irondemog_v3 happens to live
+        # in multiple questionnaire banks, thus the lookup will fail when
+        # restricted by QB.id, should the org have transitioned since the user
+        # left work incomplete from the previous protocol (TN-2747)
+        qnr = qnr.filter(QuestionnaireResponse.questionnaire_bank_id == questionnaire_bank_id)
+
     if iteration is not None:
         qnr = qnr.filter(QuestionnaireResponse.qb_iteration == iteration)
     else:

portal/models/reporting.py

@@ -6,7 +6,6 @@
 
 from flask import current_app
 from flask_babel import force_locale
-from flask_login import login_manager
 from werkzeug.exceptions import Unauthorized
 
 from ..audit import auditable_event
@@ -30,12 +29,25 @@
     qnr_csv_column_headers,
     generate_qnr_csv,
 )
-from .research_study import BASE_RS_ID, EMPRO_RS_ID
+from .research_study import BASE_RS_ID, EMPRO_RS_ID, ResearchStudy
 from .role import ROLE, Role
 from .user import User, UserRoles, patients_query
 from .user_consent import consent_withdrawal_dates
 
 
+def update_patient_adherence_data(patient_id):
+    """Cache invalidation and force rebuild for given patient's adherence data
+
+    NB - any timeline or questionnaire response data changes are invalidated and
+    updated as part of `invalidate_users_QBT()`.  This function is for edge cases
+    such as changing a user's study-id.
+    """
+    patient = User.query.get(patient_id)
+    AdherenceData.query.filter(AdherenceData.patient_id==patient_id).delete()
+    for rs_id in ResearchStudy.assigned_to(patient):
+        single_patient_adherence_data(patient_id=patient_id, research_study_id=rs_id)
+
+
 def single_patient_adherence_data(patient_id, research_study_id):
     """Update any missing (from cache) adherence data for patient
 

portal/static/js/src/admin.js

@@ -608,6 +608,7 @@ let requestTimerId = 0;
         this.filterOptionsList.forEach((o) => {
           for (const [key, values] of Object.entries(o)) {
             values.forEach((value) => {
+              if (!value[1]) return true;
               if (
                 $(
                   `#adminTable .bootstrap-table-filter-control-${key} option[value='${value[0]}']`
@@ -1412,7 +1413,7 @@ let requestTimerId = 0;
               max_attempts: 1,
             },
             function (result) {
-              if (!result?.error) self.currentTablePreference = data;
+              if (result && !result.error) self.currentTablePreference = data;
               if (callback) callback();
             }
           );

portal/templates/admin/admin_base.html

@@ -107,5 +107,6 @@
 	// placeholder variables
 	var qStatusFilterOptions = {};
 	var clinicianActionStateFilterOptions = {};
+	var visitOptions = {};
 </script>
 {%- endmacro -%}

portal/templates/admin/patients_by_org.html

@@ -56,7 +56,7 @@ <h4 class="tnth-headline">{{_("Patient List")}}</h4>
                   <th data-field="email" data-sortable="true" data-class="email-field" data-filter-control="input">{{ _("Email") }}</th>
                   {% if 'status' in config.PATIENT_LIST_ADDL_FIELDS %}
                   <th data-field="questionnaire_status" data-sortable="true" data-card-visible="false" data-searchable="true" data-width="5%" data-class="status-field" data-filter-control="select" data-filter-strict-search="true" data-filter-data="var:qStatusFilterOptions">{{ _("Questionnaire Status") }}</th>
-                  <th data-field="visit" data-sortable="true" data-card-visible="false" data-searchable="true" data-width="5%" data-class="visit-field" data-filter-control="input">{{ _("Visit") }}</th>
+                  <th data-field="visit" data-sortable="true" data-card-visible="false" data-searchable="true" data-width="5%" data-class="visit-field" data-filter-control="select" data-filter-data="var:visitOptions">{{ _("Visit") }}</th>
                   {% endif %}
                   {% if 'study_id' in config.PATIENT_LIST_ADDL_FIELDS %}<th data-field="study_id" data-sortable="true" data-searchable="true" data-class="study-id-field" data-filter-control="input" data-sorter="tnthTables.alphanumericSorter" data-width="5%">{{ _("Study ID") }}</th>{% endif %}
                   <th data-field="consentdate" data-sortable="true" data-card-visible="false" data-sorter="tnthTables.dateSorter" data-searchable="true" data-class="consentdate-field text-center">{{ _("Study Consent Date") }}</th>

portal/templates/admin/patients_substudy.html

@@ -59,7 +59,7 @@ <h2>{{list_title}}</h2>
                   <th data-field="birthdate" data-sortable="true" data-class="birthdate-field" data-visible="false">{{ _("Date of Birth") }}</th>
                   <th data-field="clinician" data-sortable="true" data-class="clinician-field" data-filter-control="input">{{ _("Treating Clinician") }}</th>
                   <th data-field="empro_status" data-sortable="true" data-card-visible="true" data-searchable="true" data-width="5%" data-class="status-field" data-filter-control="select" data-filter-strict-search="true" data-filter-data="var:qStatusFilterOptions">{{_("EMPRO Questionnaire Status")}}</th>
-                  <th data-field="empro_visit" data-sortable="true" data-card-visible="false" data-searchable="true" data-width="5%" data-class="visit-field" data-filter-control="input" data-visible="false">{{ _("Visit") }}</th>
+                  <th data-field="empro_visit" data-sortable="true" data-card-visible="false" data-searchable="true" data-width="5%" data-class="visit-field" data-filter-control="select" data-visible="false" data-filter-data="var:visitOptions">{{ _("Visit") }}</th>
                   <th data-field="action_state" data-sortable="true" data-class="intervention-actions-field" data-filter-control="select" data-filter-data="var:clinicianActionStateFilterOptions">{{ _("Clinician Action Status") }}</th>
                   <th data-field="study_id" data-sortable="true" data-searchable="true" data-class="study-id-field" data-filter-control="input" data-visible="false" data-sorter="tnthTables.alphanumericSorter" data-width="5%">{{ _("Study ID") }}</th>
                   <th data-field="empro_consentdate" data-sortable="true" data-card-visible="false" data-sorter="tnthTables.dateSorter" data-searchable="true" data-visible="false" data-class="consentdate-field text-center" data-visible="true">{{ _("Study Consent Date") }}</th>

portal/views/demographics.py

@@ -7,7 +7,9 @@
 from ..database import db
 from ..extensions import oauth
 from ..models.reference import MissingReference
+from ..models.reporting import update_patient_adherence_data
 from ..models.user import current_user, get_user
+from ..models.role import ROLE
 from .crossdomain import crossdomain
 
 demographics_api = Blueprint('demographics_api', __name__, url_prefix='/api')
@@ -176,7 +178,9 @@ def demographics_set(patient_id):
         patient_id, json.dumps(request.json)), user_id=current_user().id,
         subject_id=patient_id, context='user')
 
-    # update the patient_table cache with any change from above
-    patient_list_update_patient(patient_id)
+    # update the respective cache tables with any change from above
+    if patient.has_role(ROLE.PATIENT.value):
+        patient_list_update_patient(patient_id)
+        update_patient_adherence_data(patient_id)
 
     return jsonify(patient.as_fhir(include_empties=False))

portal/views/patient.py

@@ -328,7 +328,7 @@ def patient_timeline(patient_id):
     )
     from ..models.qbd import QBD
     from ..models.qb_status import QB_Status
-    from ..models.questionnaire_bank import visit_name
+    from ..models.questionnaire_bank import translate_visit_name, visit_name
     from ..models.questionnaire_response import aggregate_responses
     from ..models.research_protocol import ResearchProtocol
     from ..tasks import cache_single_patient_adherence_data
@@ -392,7 +392,7 @@ def patient_timeline(patient_id):
                 'at': FHIR_datetime.as_fhir(qbt.at),
                 'qb (id, iteration)': "{} ({}, {})".format(
                     qbd.questionnaire_bank.name, qbd.qb_id, qbd.iteration),
-                'visit': visit_name(qbd)}
+                'visit': translate_visit_name(visit_name(qbd))}
             if qbt.status == OverallStatus.due:
                 data['questionnaires'] = ','.join(
                     [q.name for q in qbd.questionnaire_bank.questionnaires])

portal/views/patients.py

@@ -17,6 +17,7 @@
 from ..models.intervention import Intervention
 from ..models.organization import Organization, OrgTree
 from ..models.patient_list import PatientList
+from ..models.questionnaire_bank import translate_visit_name
 from ..models.qb_status import patient_research_study_status
 from ..models.role import ROLE
 from ..models.research_study import EMPRO_RS_ID, ResearchStudy
@@ -181,11 +182,25 @@ def requested_orgs(user, research_study_id):
         distinct_action = PatientList.query.distinct(PatientList.action_state).with_entities(
             PatientList.action_state)
         options.append({"action_state": [(state[0], _(state[0])) for state in distinct_action]})
+        distinct_visits = PatientList.query.distinct(PatientList.empro_visit).with_entities(
+            PatientList.empro_visit)
+        sorted_visits = sorted(
+            [v[0] for v in distinct_visits if v[0]],
+            key=lambda x: (0 if not x.split()[-1].isdigit() else int(x.split()[-1]))
+        )
+        options.append({"empro_visit": [(visit, translate_visit_name(visit)) for visit in sorted_visits]})
     else:
         distinct_status = PatientList.query.distinct(
             PatientList.questionnaire_status).with_entities(PatientList.questionnaire_status)
         options.append(
             {"questionnaire_status": [(status[0], _(status[0])) for status in distinct_status]})
+        distinct_visits = PatientList.query.distinct(PatientList.visit).with_entities(
+            PatientList.visit)
+        sorted_visits = sorted(
+            [v[0] for v in distinct_visits if v[0]],
+            key=lambda x: (0 if not x.split()[-1].isdigit() else int(x.split()[-1]))
+        )
+        options.append({"visit": [(visit, translate_visit_name(visit)) for visit in sorted_visits]})
 
     viewable_orgs = requested_orgs(user, research_study_id)
     query = PatientList.query.filter(PatientList.org_id.in_(viewable_orgs))
@@ -222,8 +237,8 @@ def requested_orgs(user, research_study_id):
             "questionnaire_status": _(row.questionnaire_status),
             "empro_status": _(row.empro_status),
             "action_state": _(row.action_state),
-            "visit": row.visit,
-            "empro_visit": row.empro_visit,
+            "visit": translate_visit_name(row.visit),
+            "empro_visit": translate_visit_name(row.empro_visit),
             "study_id": row.study_id,
             "consentdate": row.consentdate,
             "empro_consentdate": row.empro_consentdate,