Merge branch 'release/v24.10.10'

docker/docker-compose.yaml

@@ -88,7 +88,7 @@ services:
     image: redis
 
   db:
-    image: postgres:${POSTGRES_VERSION:-13}
+    image: postgres:${POSTGRES_VERSION:-16}
     environment:
       # use generic postgres env vars to configure env vars specific to dockerized postgres
       POSTGRES_DB: ${PGDATABASE:-portaldb}

manage.py

@@ -756,8 +756,8 @@ def find_overlaps(correct_overlaps, reprocess_qnrs):
                     acting_user_id=admin.id,
                 )
 
-            update_users_QBT(
-                patient.id, research_study_id=0, invalidate_existing=True)
+            invalidate_users_QBT(user_id=patient.id, research_study_id=0)
+            update_users_QBT(user_id=patient.id, research_study_id=0)
             present_before_after_state(
                 patient.id, patient.external_study_id, b4)
 
@@ -838,8 +838,8 @@ def preview_site_update(org_id, retired):
             research_study_id=0,
             acting_user_id=admin.id,
         )
-        update_users_QBT(
-            patient.id, research_study_id=0, invalidate_existing=True)
+        invalidate_users_QBT(user_id=patient.id, research_study_id=0)
+        update_users_QBT(user_id=patient.id, research_study_id=0)
         after_qnrs, after_timeline, qnrs_lost_reference, _ = present_before_after_state(
             patient.id, patient.external_study_id, patient_state[patient.id])
         total_qnrs += len(patient_state[patient.id]['qnrs'])

portal/config/eproms/ScheduledJob.json

@@ -276,6 +276,14 @@
       "schedule": "0 0 0 0 0",
       "task": "raise_background_exception_task"
     },
+    {
+      "active": false,
+      "args": null,
+      "name": "Populate Patient List",
+      "resourceType": "ScheduledJob",
+      "schedule": "0 0 0 0 0",
+      "task": "cache_patient_list"
+    },
     {
       "active": true,
       "args": null,
@@ -300,6 +308,15 @@
       "resourceType": "ScheduledJob",
       "schedule": "30 14 * * *",
       "task": "cache_adherence_data_task"
+    },
+    {
+      "active": true,
+      "args": null,
+      "kwargs": null,
+      "name": "Cache Research Report Data",
+      "resourceType": "ScheduledJob",
+      "schedule": "30 23 * * *",
+      "task": "cache_research_data_task"
     }
   ],
   "id": "SitePersistence v0.2",

portal/eproms/templates/eproms/assessment_engine/ae_base.html

@@ -5,3 +5,18 @@
     {%- block body -%}{%- endblock -%}
 </section>
 <div class="loading-container hide"><div class="content"><i class="fa fa-spinner fa-spin fa-2x"></i>{{_("Loading")}} ...</div></div>
+
+<div class="wait-indicator-wrapper hide">
+    <div class="wait-indicator-container">
+        <div class="frame">
+        <div class="top"></div>
+        <div class="bottom">
+            <div class="drip"></div>
+            <div class="blob"></div>
+            <div class="glass"></div>
+        </div>
+        </div>
+        <!-- loading text is pre-existing -->
+        <div class="loading-text-container">{{_("Loading")}} ...</div>
+    </div>
+</div>

portal/migrations/versions/038a1a5f4218_.py

@@ -0,0 +1,76 @@
+"""add patient_list table for paginated /patients view
+
+Revision ID: 038a1a5f4218
+Revises: daee63f50d35
+Create Date: 2024-09-30 16:10:26.216512
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '038a1a5f4218'
+down_revision = 'daee63f50d35'
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('patient_list',
+    sa.Column('userid', sa.Integer(), nullable=False),
+    sa.Column('study_id', sa.Text(), nullable=True),
+    sa.Column('firstname', sa.String(length=64), nullable=True),
+    sa.Column('lastname', sa.String(length=64), nullable=True),
+    sa.Column('birthdate', sa.Date(), nullable=True),
+    sa.Column('email', sa.String(length=120), nullable=True),
+    sa.Column('questionnaire_status', sa.Text(), nullable=True),
+    sa.Column('empro_status', sa.Text(), nullable=True),
+    sa.Column('clinician', sa.Text(), nullable=True),
+    sa.Column('action_state', sa.Text(), nullable=True),
+    sa.Column('visit', sa.Text(), nullable=True),
+    sa.Column('empro_visit', sa.Text(), nullable=True),
+    sa.Column('consentdate', sa.DateTime(), nullable=True),
+    sa.Column('empro_consentdate', sa.DateTime(), nullable=True),
+    sa.Column('org_name', sa.Text(), nullable=True),
+    sa.Column('deleted', sa.Boolean(), nullable=True),
+    sa.Column('test_role', sa.Boolean(), nullable=True),
+    sa.Column('org_id', sa.Integer(), nullable=True),
+    sa.Column('last_updated', sa.DateTime(), nullable=False),
+    sa.ForeignKeyConstraint(['org_id'], ['organizations.id'], ),
+    sa.PrimaryKeyConstraint('userid')
+    )
+    op.create_index(op.f('ix_patient_list_action_state'), 'patient_list', ['action_state'], unique=False)
+    op.create_index(op.f('ix_patient_list_birthdate'), 'patient_list', ['birthdate'], unique=False)
+    op.create_index(op.f('ix_patient_list_clinician'), 'patient_list', ['clinician'], unique=False)
+    op.create_index(op.f('ix_patient_list_consentdate'), 'patient_list', ['consentdate'], unique=False)
+    op.create_index(op.f('ix_patient_list_email'), 'patient_list', ['email'], unique=False)
+    op.create_index(op.f('ix_patient_list_empro_consentdate'), 'patient_list', ['empro_consentdate'], unique=False)
+    op.create_index(op.f('ix_patient_list_empro_status'), 'patient_list', ['empro_status'], unique=False)
+    op.create_index(op.f('ix_patient_list_empro_visit'), 'patient_list', ['empro_visit'], unique=False)
+    op.create_index(op.f('ix_patient_list_firstname'), 'patient_list', ['firstname'], unique=False)
+    op.create_index(op.f('ix_patient_list_lastname'), 'patient_list', ['lastname'], unique=False)
+    op.create_index(op.f('ix_patient_list_org_name'), 'patient_list', ['org_name'], unique=False)
+    op.create_index(op.f('ix_patient_list_questionnaire_status'), 'patient_list', ['questionnaire_status'], unique=False)
+    op.create_index(op.f('ix_patient_list_study_id'), 'patient_list', ['study_id'], unique=False)
+    op.create_index(op.f('ix_patient_list_visit'), 'patient_list', ['visit'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_patient_list_visit'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_study_id'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_questionnaire_status'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_org_name'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_lastname'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_firstname'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_empro_visit'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_empro_status'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_empro_consentdate'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_email'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_consentdate'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_clinician'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_birthdate'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_action_state'), table_name='patient_list')
+    op.drop_table('patient_list')
+    # ### end Alembic commands ###

portal/migrations/versions/5a300be640fb_.py

@@ -0,0 +1,52 @@
+"""empty message
+
+Revision ID: 5a300be640fb
+Revises: 038a1a5f4218
+Create Date: 2024-10-08 14:34:28.085963
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '5a300be640fb'
+down_revision = '038a1a5f4218'
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint('adherence_data_patient_id_fkey', 'adherence_data')
+    op.create_foreign_key(
+        'adherence_data_patient_id_fkey',
+        'adherence_data',
+        'users', ['patient_id'], ['id'], ondelete='cascade')
+    op.alter_column('patient_list', 'last_updated',
+               existing_type=postgresql.TIMESTAMP(),
+               nullable=True)
+    op.create_foreign_key(
+        'patient_list_userid_fkey',
+        'patient_list',
+        'users', ['userid'], ['id'], ondelete='cascade')
+    op.drop_constraint('research_data_subject_id_fkey', 'research_data', type_='foreignkey')
+    op.create_foreign_key(
+        'research_data_subject_id_fkey',
+        'research_data',
+        'users', ['subject_id'], ['id'], ondelete='cascade')
+    op.drop_constraint('research_data_questionnaire_response_id_fkey', 'research_data', type_='foreignkey')
+    op.create_foreign_key(
+        'research_data_questionnaire_response_id_fkey',
+        'research_data',
+        'questionnaire_responses', ['questionnaire_response_id'], ['id'], ondelete='cascade')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint('research_data_subject_id_fkey', 'research_data', type_='foreignkey')
+    op.create_foreign_key('research_data_subject_id_fkey', 'research_data', 'users', ['subject_id'], ['id'])
+    op.drop_constraint('patient_list_userid_fkey', 'patient_list', type_='foreignkey')
+    op.alter_column('patient_list', 'last_updated',
+               existing_type=postgresql.TIMESTAMP(),
+               nullable=False)
+    # ### end Alembic commands ###

portal/migrations/versions/daee63f50d35_.py

@@ -0,0 +1,57 @@
+"""Add research_data table, to hold questionnaire response research data in a cache
+
+Revision ID: daee63f50d35
+Revises: cf586ed4f043
+Create Date: 2024-05-21 17:00:58.204998
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'daee63f50d35'
+down_revision = '6120fcfc474a'
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        'research_data',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('subject_id', sa.Integer(), nullable=False),
+        sa.Column('questionnaire_response_id', sa.Integer(), nullable=False),
+        sa.Column('instrument', sa.Text(), nullable=False),
+        sa.Column('research_study_id', sa.Integer(), nullable=False),
+        sa.Column('authored', sa.DateTime(), nullable=False),
+        sa.Column('data', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+        sa.ForeignKeyConstraint(['subject_id'], ['users.id'], ),
+        sa.ForeignKeyConstraint(
+            ['questionnaire_response_id'], ['questionnaire_responses.id'], ),
+        sa.PrimaryKeyConstraint('id'),
+        )
+
+    op.create_index(
+        op.f('ix_research_data_authored'), 'research_data', ['authored'], unique=False)
+    op.create_index(
+        op.f('ix_research_data_instrument'), 'research_data', ['instrument'], unique=False)
+    op.create_index(
+        op.f('ix_research_data_subject_id'), 'research_data', ['subject_id'], unique=False)
+    op.create_index(
+        op.f('ix_research_data_questionnaire_response_id'),
+        'research_data', ['questionnaire_response_id'], unique=True)
+    op.create_index(
+        op.f('ix_research_data_research_study_id'),
+        'research_data', ['research_study_id'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_research_data_research_study_id'), table_name='research_data')
+    op.drop_index(op.f('ix_research_data_questionnaire_response_id'), table_name='research_data')
+    op.drop_index(op.f('ix_research_data_subject_id'), table_name='research_data')
+    op.drop_index(op.f('ix_research_data_instrument'), table_name='research_data')
+    op.drop_index(op.f('ix_research_data_authored'), table_name='research_data')
+    op.drop_table('research_data')
+    # ### end Alembic commands ###

portal/models/adherence_data.py

@@ -23,7 +23,8 @@ class AdherenceData(db.Model):
     """
     __tablename__ = 'adherence_data'
     id = db.Column(db.Integer, primary_key=True)
-    patient_id = db.Column(db.ForeignKey('users.id'), index=True, nullable=False)
+    patient_id = db.Column(
+        db.ForeignKey('users.id', ondelete='cascade'), index=True, nullable=False)
     rs_id_visit = db.Column(
         db.Text, index=True, nullable=False,
         doc="rs_id:visit_name")
@@ -77,12 +78,20 @@ def persist(patient_id, rs_id_visit, valid_for_days, data):
             except TypeError:
                 raise ValueError(f"couldn't encode {k}:{v}, {type(v)}")
 
-        record = AdherenceData(
-            patient_id=patient_id,
-            rs_id_visit=rs_id_visit,
-            valid_till=valid_till,
-            data=data)
-        db.session.add(record)
+        # only a single row for a given patient, rs_id_visit allowed.  replace or add
+        record = AdherenceData.query.filter(
+            AdherenceData.patient_id == patient_id).filter(
+            AdherenceData.rs_id_visit == rs_id_visit).first()
+        if record:
+            record.valid_till = valid_till
+            record.data = data
+        else:
+            record = AdherenceData(
+                patient_id=patient_id,
+                rs_id_visit=rs_id_visit,
+                valid_till=valid_till,
+                data=data)
+            db.session.add(record)
         db.session.commit()
         return db.session.merge(record)