Merge branch 'release/v23.7.24'

docker/docker-compose.dev.yaml

@@ -16,4 +16,21 @@ services:
       - source: ../
         target: /mnt/code
         type: bind
-        read_only: true
+
+  celeryworker:
+    environment:
+      FLASK_DEBUG: 1
+      FLASK_APP: /mnt/code/manage.py
+    volumes:
+      - source: ../
+        target: /mnt/code
+        type: bind
+
+  celeryworkerslow:
+    environment:
+      FLASK_DEBUG: 1
+      FLASK_APP: /mnt/code/manage.py
+    volumes:
+      - source: ../
+        target: /mnt/code
+        type: bind

docker/docker-compose.prod.yaml

@@ -29,11 +29,13 @@ services:
     # Set lower CPU priority to prevent blocking web service
     cap_add:
       - SYS_NICE
-    command:
+    command: sh -c '
+      flask generate-site-cfg &&
       nice
         celery worker
           --app portal.celery_worker.celery
           --loglevel debug
+      '
 
   celeryworkerslow:
     restart: unless-stopped
@@ -44,12 +46,14 @@ services:
     # Set lower CPU priority to prevent blocking web service
     cap_add:
       - SYS_NICE
-    command:
+    command: sh -c '
+      flask generate-site-cfg &&
       nice -n 15
         celery worker
           --app portal.celery_worker.celery
           --queue low_priority
           --loglevel debug
+      '
 
   celerybeat:
     restart: unless-stopped

docker/docker-compose.yaml

@@ -42,20 +42,24 @@ services:
 
   celeryworker:
     <<: *service_base
-    command:
+    command: sh -c '
+      flask generate-site-cfg &&
       celery worker
         --app portal.celery_worker.celery
         --loglevel debug
+      '
     depends_on:
       - redis
 
   celeryworkerslow:
     <<: *service_base
-    command:
+    command: sh -c '
+      flask generate-site-cfg &&
       celery worker
         --app portal.celery_worker.celery
         --queue low_priority
         --loglevel debug
+      '
     depends_on:
       - redis
 

manage.py

@@ -21,6 +21,7 @@
 
 from portal.audit import auditable_event
 from portal.date_tools import FHIR_datetime
+from portal.config.config_persistence import import_config
 from portal.config.site_persistence import SitePersistence
 from portal.extensions import db, user_manager
 from portal.factories.app import create_app
@@ -178,6 +179,20 @@ def seed(keep_unmentioned=False):
         keep_unmentioned=keep_unmentioned)
 
 
+@app.cli.command()
+def generate_site_cfg():
+    """Generate only the site.cfg file via site_persistence
+
+    Typically done via `sync` or `seed`, this option exists for the
+    backend job queues to generate the `site.cfg` file to maintain
+    consistent configuration with the front end, withou the overhead
+    of the rest of `sync`
+    """
+    app.logger.info("generate-site-cfg begin")
+    import_config(target_dir=None)
+    app.logger.info("generate-site-cfg complete")
+
+
 @click.option('--directory', '-d', default=None, help="Export directory")
 @click.option(
     '--staging_exclusion', '-x', default=False, is_flag=True,

portal/config/eproms/ScheduledJob.json

@@ -275,6 +275,31 @@
       "resourceType": "ScheduledJob",
       "schedule": "0 0 0 0 0",
       "task": "raise_background_exception_task"
+    },
+    {
+      "active": true,
+      "args": null,
+      "kwargs": {
+        "include_test_role": true,
+        "research_study_id": 0,
+        "limit": 5000
+      },
+      "name": "Cache Adherence Data -- Global Study",
+      "resourceType": "ScheduledJob",
+      "schedule": "30 21 * * *",
+      "task": "cache_adherence_data_task"
+    },
+    {
+      "active": true,
+      "args": null,
+      "kwargs": {
+        "include_test_role": true,
+        "research_study_id": 1
+      },
+      "name": "Cache Adherence Data -- EMPRO Study",
+      "resourceType": "ScheduledJob",
+      "schedule": "30 14 * * *",
+      "task": "cache_adherence_data_task"
     }
   ],
   "id": "SitePersistence v0.2",

portal/factories/app.py

@@ -137,6 +137,9 @@ def configure_app(app, config):
     """Load successive configs - overriding defaults"""
     app.config.from_object(DefaultConfig)
     app.config.from_pyfile('base.cfg', silent=True)
+    site_cfg_fullpath = os.path.join(app.config.root_path, SITE_CFG)
+    if not os.path.isfile(site_cfg_fullpath):
+        app.logger.warning(f"{site_cfg_fullpath} not found!")
     app.config.from_pyfile(SITE_CFG, silent=True)
     app.config.from_pyfile('application.cfg', silent=True)
 

portal/migrations/versions/2e9b9e696bb8_adherencedata_table.py

@@ -0,0 +1,41 @@
+"""AdherenceData table
+
+Revision ID: 2e9b9e696bb8
+Revises: ccb67176c56f
+Create Date: 2023-06-15 17:45:43.699277
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '2e9b9e696bb8'
+down_revision = 'ccb67176c56f'
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('adherence_data',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('patient_id', sa.Integer(), nullable=False),
+    sa.Column('rs_id_visit', sa.Text(), nullable=False),
+    sa.Column('valid_till', sa.DateTime(), nullable=False),
+    sa.Column('data', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    sa.ForeignKeyConstraint(['patient_id'], ['users.id'], ),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('patient_id', 'rs_id_visit', name='_adherence_unique_patient_visit')
+    )
+    op.create_index(op.f('ix_adherence_data_patient_id'), 'adherence_data', ['patient_id'], unique=False)
+    op.create_index(op.f('ix_adherence_data_rs_id_visit'), 'adherence_data', ['rs_id_visit'], unique=False)
+    op.create_index(op.f('ix_adherence_data_valid_till'), 'adherence_data', ['valid_till'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_adherence_data_valid_till'), table_name='adherence_data')
+    op.drop_index(op.f('ix_adherence_data_rs_id_visit'), table_name='adherence_data')
+    op.drop_index(op.f('ix_adherence_data_patient_id'), table_name='adherence_data')
+    op.drop_table('adherence_data')
+    # ### end Alembic commands ###

portal/models/adherence_data.py

@@ -0,0 +1,117 @@
+""" model data for adherence reports """
+from datetime import datetime, timedelta
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy import UniqueConstraint
+
+from ..database import db
+
+
+class AdherenceData(db.Model):
+    """ Cached adherence report data
+
+    Full history adherence data is expensive to generate, retain between reports.
+    Cache reportable data in simple JSON structure, maintaining keys for lookup
+    and invalidation timestamps.
+
+    rs_id_visit: the numeric rs_id and visit month string joined with a colon
+    valid_till: old history data never changes, unless an external event such
+        as a user's consent date or organization research protocol undergoes
+        change.  active visits require more frequent updates but are considered
+        fresh enough for days.  client code sets valid_till as appropriate.
+
+    """
+    __tablename__ = 'adherence_data'
+    id = db.Column(db.Integer, primary_key=True)
+    patient_id = db.Column(db.ForeignKey('users.id'), index=True, nullable=False)
+    rs_id_visit = db.Column(
+        db.Text, index=True, nullable=False,
+        doc="rs_id:visit_name")
+    valid_till = db.Column(
+        db.DateTime, nullable=False, index=True,
+        doc="cached values good till time passed")
+    data = db.Column(JSONB)
+
+    __table_args__ = (UniqueConstraint(
+        'patient_id', 'rs_id_visit', name='_adherence_unique_patient_visit'),)
+
+    @staticmethod
+    def rs_visit_string(rs_id, visit_string):
+        """trivial helper to build rs_id_visit string into desired format"""
+        assert isinstance(rs_id, int)
+        assert visit_string
+        return f"{rs_id}:{visit_string}"
+
+    def rs_visit_parse(self):
+        """break parts of rs_id and visit_string out of rs_id_visit field"""
+        rs_id, visit_string = self.rs_id_visit.split(':')
+        assert visit_string
+        return int(rs_id), visit_string
+
+    @staticmethod
+    def fetch(patient_id, rs_id_visit):
+        """shortcut for common lookup need
+
+        :return: populated AdherenceData instance if found, None otherwise
+        """
+        result = AdherenceData.query.filter(
+            AdherenceData.patient_id == patient_id).filter(
+            AdherenceData.rs_id_visit == rs_id_visit).first()
+        if result:
+            assert result.valid_till > datetime.utcnow()
+        return result
+
+    @staticmethod
+    def persist(patient_id, rs_id_visit, valid_for_days, data):
+        """shortcut to persist a row, returns new instance"""
+        import json
+        valid_till = datetime.utcnow() + timedelta(days=valid_for_days)
+        for k, v in data.items():
+            try:
+                json.dumps(k)
+                json.dumps(v)
+            except TypeError:
+                raise ValueError(f"couldn't encode {k}:{v}, {type(v)}")
+
+        record = AdherenceData(
+            patient_id=patient_id,
+            rs_id_visit=rs_id_visit,
+            valid_till=valid_till,
+            data=data)
+        db.session.add(record)
+        db.session.commit()
+        return db.session.merge(record)
+
+
+def sort_by_visit_key(d):
+    """Given dict returns ordered list of values sorted by key
+
+    Sort by key using the following rules:
+    "Baseline" comes first
+    "Indefinite" comes last
+    "Month <n>" in the middle, sorted by integer value
+
+    :returns: list of values sorted by keys
+    """
+    def sort_key(key):
+        if key == 'Baseline':
+            return 0, 0
+        elif key == 'Indefinite':
+            return 2, 0
+        else:
+            month, num = key.split(" ")
+            assert month == "Month"
+            return 1, int(num)
+
+    sorted_keys = sorted(d.keys(), key=sort_key)
+    sorted_values = [d[key] for key in sorted_keys]
+    return sorted_values
+
+
+def sorted_adherence_data(patient_id, research_study_id):
+    """Shortcut to obtain ordered list for given patient:research_study"""
+    rows = AdherenceData.query.filter(
+        AdherenceData.patient_id == patient_id).filter(
+        AdherenceData.rs_id_visit.like(f"{research_study_id}%"))
+    # Sort locally, given complicated sort function
+    presorted = {row.rs_id_visit.split(':')[1]: row.data for row in rows}
+    return sort_by_visit_key(presorted)

portal/models/organization.py

@@ -43,6 +43,9 @@
 @cache.memoize(timeout=FIVE_MINS)
 def org_country(org_id):
     """Cache enabled country lookup for given organization ID"""
+    if org_id == 0:  # None of the above, happens on testing
+        return None
+
     ot = OrgTree()
     org_ids = ot.at_and_above_ids(org_id)
     for org in Organization.query.filter(
@@ -214,7 +217,7 @@ def sitecode(self):
         """Return site code identifier if found, else empty string"""
         system = current_app.config.get('REPORTING_IDENTIFIER_SYSTEMS')
         if not system:
-            return ""
+            return "REPORTING_IDENTIFIER_SYSTEMS not defined"
         if isinstance(system, (list, tuple)):
             # catch need to support more than one
             assert len(system) == 1