From 72760e9c7fe6b8c31a30b0680cf013416cab2cef Mon Sep 17 00:00:00 2001 From: Paul Bugni Date: Mon, 11 Dec 2023 16:23:56 -0800 Subject: [PATCH 01/12] EMPRO expired status is ambiguous - can mean "not yet available" or "expired". Added comparison with consent to correct. --- portal/models/reporting.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/portal/models/reporting.py b/portal/models/reporting.py index af9ea6e40..6e9b54ed3 100644 --- a/portal/models/reporting.py +++ b/portal/models/reporting.py @@ -152,10 +152,14 @@ def empro_row_detail(row, ts_reporting): # build up data until we find valid cache for patient's history status = str(qb_stats.overall_status) row = patient_data(patient) + row["status"] = status if status == "Expired" and research_study_id == EMPRO_RS_ID: - row["status"] = "Not Yet Available" - else: - row["status"] = status + # Expired status ambiguous for EMPRO - either not available + # due to complex business rules around start or walked off + # the end. Assume if consent + 1year > now, it's the former. + consent = datetime.strptime(row["consent"], "%d-%b-%Y %H:%M:%S") + if consent + timedelta(days=365) > as_of_date: + row["status"] = "Not Yet Available" if last_viable: general_row_detail(row, patient, last_viable) From 52b7dff93e88ea6b43a0060fc0add5dca204dd42 Mon Sep 17 00:00:00 2001 From: Paul Bugni Date: Mon, 11 Dec 2023 19:31:43 -0800 Subject: [PATCH 02/12] migration to purge bad adherence cache data for affected patients. --- portal/migrations/versions/66368e673005_.py | 63 +++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 portal/migrations/versions/66368e673005_.py diff --git a/portal/migrations/versions/66368e673005_.py b/portal/migrations/versions/66368e673005_.py new file mode 100644 index 000000000..fb788ac1b --- /dev/null +++ b/portal/migrations/versions/66368e673005_.py @@ -0,0 +1,63 @@ +"""IRONN-225 update adherence data for expired EMPRO users + +Revision ID: 66368e673005 +Revises: d1f3ed8d16ef +Create Date: 2023-12-11 16:56:10.427854 + +""" +from alembic import op +from datetime import datetime +import sqlalchemy as sa +from sqlalchemy.orm import sessionmaker + + +# revision identifiers, used by Alembic. +revision = '66368e673005' +down_revision = 'd1f3ed8d16ef' + +Session = sessionmaker() + + +def upgrade(): + # IRONN-225 noted expired EMPRO users adherence data showed + # `not yet available`. Code corrected, need to force renewal + # for those affected. + + bind = op.get_bind() + session = Session(bind=bind) + + now = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") + patient_ids = [] + # get list of non-deleted users with a 12th month expiration + # that has already passed. (12 = baseline + zero-index 10) + for patient_id in session.execute( + "SELECT DISTINCT(user_id) FROM qb_timeline JOIN users" + " ON users.id = user_id WHERE deleted_id IS NULL" + " AND research_study_id = 1 AND qb_iteration = 10" + f" AND status = 'expired' AND at < '{now}'"): + patient_ids.append(patient_id[0]) + + # purge their respective rows from adherence cache, IFF status + # shows IRONN-225 symptom. + rs_visit = "1:Month 12" + for patient_id in patient_ids: + status = session.execute( + "SELECT data->>'status' FROM adherence_data WHERE" + f" patient_id = {patient_id} AND" + f" rs_id_visit = '{rs_visit}'" + ).first() + if status and status[0] != "Not Yet Available": + continue + + # purge the user's EMPRO adherence rows to force refresh + session.execute( + "DELETE FROM adherence_data WHERE" + f" patient_id = {patient_id} AND" + f" rs_id_visit like '1:%'" + ) + + +def downgrade(): + # No reasonable downgrade + pass + From c04cd1c31a4a6c82b226b4db1747e3945aa8184d Mon Sep 17 00:00:00 2001 From: Paul Bugni Date: Tue, 12 Dec 2023 16:51:05 -0800 Subject: [PATCH 03/12] use a far more reasonable 5 min timeout default for "CacheModeration" - believed this may be part of missing results from IRONN-222 --- portal/timeout_lock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portal/timeout_lock.py b/portal/timeout_lock.py index 75f91a6a3..715f7f2bf 100644 --- a/portal/timeout_lock.py +++ b/portal/timeout_lock.py @@ -106,7 +106,7 @@ def guarded_task_launch(task, **kwargs): class CacheModeration(object): """Redis key implementation to prevent same key from excessive updates""" - def __init__(self, key, timeout=3600): + def __init__(self, key, timeout=300): self.key = key self.timeout = timeout self.redis = redis.StrictRedis.from_url( From eb0ab6e2ca0fb40b4512303f8c7a6e4a25261977 Mon Sep 17 00:00:00 2001 From: Paul Bugni Date: Tue, 12 Dec 2023 16:52:01 -0800 Subject: [PATCH 04/12] force a re-population of adherence_data on /timeline purge (or if missing) --- portal/views/patient.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/portal/views/patient.py b/portal/views/patient.py index 09a39ce89..b2d7d9b77 100644 --- a/portal/views/patient.py +++ b/portal/views/patient.py @@ -30,7 +30,8 @@ from ..models.questionnaire_bank import QuestionnaireBank, trigger_date from ..models.questionnaire_response import QuestionnaireResponse from ..models.reference import Reference -from ..models.research_study import ResearchStudy +from ..models.reporting import single_patient_adherence_data +from ..models.research_study import EMPRO_RS_ID, ResearchStudy from ..models.role import ROLE from ..models.user import User, current_user, get_user from ..timeout_lock import ADHERENCE_DATA_KEY, CacheModeration @@ -341,6 +342,7 @@ def patient_timeline(patient_id): # questionnaire_response : qb relationships and remove cache lock # on adherence data. if purge == 'all': + # remove adherence cache key to allow fresh run cache_moderation = CacheModeration(key=ADHERENCE_DATA_KEY.format( patient_id=patient_id, research_study_id=research_study_id)) @@ -455,6 +457,13 @@ def get_recur_id(qnr): status['indefinite status'] = indef_status adherence_data = sorted_adherence_data(patient_id, research_study_id) + if not adherence_data: + # immediately following a cache purge, adherence data is gone and + # needs to be recreated. + now = datetime.utcnow() + single_patient_adherence_data( + user, as_of_date=now, research_study_id=EMPRO_RS_ID) + adherence_data = sorted_adherence_data(patient_id, research_study_id) if trace: return jsonify( From 541a12db99c6fd49b72bc283b625f9564331e7db Mon Sep 17 00:00:00 2001 From: Paul Bugni Date: Thu, 14 Dec 2023 16:10:45 -0800 Subject: [PATCH 05/12] reduce adherence data cache valid periods to more reasonable times. doesn't take that long to update, good to catch any problems with a re-calc on a more regular basis. --- portal/models/reporting.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/portal/models/reporting.py b/portal/models/reporting.py index 6e9b54ed3..2566a4a06 100644 --- a/portal/models/reporting.py +++ b/portal/models/reporting.py @@ -168,8 +168,8 @@ def empro_row_detail(row, ts_reporting): ts_reporting = TriggerStatesReporting(patient_id=patient.id) empro_row_detail(row, ts_reporting) - # latest is only valid for a week, unless the user withdrew - valid_for = 500 if row['status'] in ('Expired', 'Withdrawn') else 7 + # latest is only valid for a day, unless the user withdrew + valid_for = 30 if row['status'] in ('Expired', 'Withdrawn') else 1 AdherenceData.persist( patient_id=patient.id, rs_id_visit=rs_visit, @@ -181,9 +181,10 @@ def empro_row_detail(row, ts_reporting): for qbd, status in qb_stats.older_qbds(last_viable): rs_visit = AdherenceData.rs_visit_string( research_study_id, visit_name(qbd)) - # once we find cached_data, the rest of the user's history is good + # once we find cached_data, the rest of the user's history is likely + # good, but best to verify nothing is stale if AdherenceData.fetch(patient_id=patient.id, rs_id_visit=rs_visit): - break + continue historic = row.copy() historic['status'] = status @@ -194,7 +195,7 @@ def empro_row_detail(row, ts_reporting): AdherenceData.persist( patient_id=patient.id, rs_id_visit=rs_visit, - valid_for_days=500, + valid_for_days=30, data=historic) added_rows += 1 @@ -227,7 +228,7 @@ def empro_row_detail(row, ts_reporting): AdherenceData.persist( patient_id=patient.id, rs_id_visit=rs_visit, - valid_for_days=500, + valid_for_days=30, data=indef) added_rows += 1 From 617a1f3949d44b040a15d6b01c89a16256ef0906 Mon Sep 17 00:00:00 2001 From: Paul Bugni Date: Thu, 14 Dec 2023 16:12:52 -0800 Subject: [PATCH 06/12] extend function to build questionnaire report to take a list of patients. enables quick, small batch runs, and use from /timeline corrected an old bug in view lookup due to scope problem. --- portal/models/questionnaire_response.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/portal/models/questionnaire_response.py b/portal/models/questionnaire_response.py index 208413581..77ba003f5 100644 --- a/portal/models/questionnaire_response.py +++ b/portal/models/questionnaire_response.py @@ -842,7 +842,7 @@ def required_qs(self, qb_id): def aggregate_responses( instrument_ids, current_user, research_study_id, patch_dstu2=False, - ignore_qb_requirement=False, celery_task=None): + ignore_qb_requirement=False, celery_task=None, patient_ids=None): """Build a bundle of QuestionnaireResponses :param instrument_ids: list of instrument_ids to restrict results to @@ -852,13 +852,17 @@ def aggregate_responses( :param patch_dstu2: set to make bundle DSTU2 compliant :param ignore_qb_requirement: set to include all questionnaire responses :param celery_task: if defined, send occasional progress updates + :param patient_ids: if defined, limit result set to given patient list """ from .qb_timeline import qb_status_visit_name # avoid cycle # Gather up the patient IDs for whom current user has 'view' permission user_ids = patients_query( - current_user, include_test_role=False).with_entities(User.id) + current_user, + include_test_role=False, + filter_by_ids=patient_ids, + ).with_entities(User.id) annotated_questionnaire_responses = [] questionnaire_responses = QuestionnaireResponse.query.filter( @@ -920,7 +924,7 @@ def aggregate_responses( 'resource': document, # Todo: return URL to individual QuestionnaireResponse resource 'fullUrl': url_for( - '.assessment', + 'assessment_engine_api.assessment', patient_id=subject.id, _external=True, ), From ccb6b6ef59cc838241ca9d52e1c54961d87ce950 Mon Sep 17 00:00:00 2001 From: Paul Bugni Date: Thu, 14 Dec 2023 16:14:34 -0800 Subject: [PATCH 07/12] extend /timeline to include questionnaire report data for patient. obtain adherence data if not already in cache on /timeline request --- portal/views/patient.py | 45 ++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/portal/views/patient.py b/portal/views/patient.py index b2d7d9b77..3fced522a 100644 --- a/portal/views/patient.py +++ b/portal/views/patient.py @@ -324,6 +324,7 @@ def patient_timeline(patient_id): from ..models.qbd import QBD from ..models.qb_status import QB_Status from ..models.questionnaire_bank import visit_name + from ..models.questionnaire_response import aggregate_responses from ..models.research_protocol import ResearchProtocol from ..trace import dump_trace, establish_trace @@ -465,17 +466,41 @@ def get_recur_id(qnr): user, as_of_date=now, research_study_id=EMPRO_RS_ID) adherence_data = sorted_adherence_data(patient_id, research_study_id) + qnr_responses = aggregate_responses( + instrument_ids=None, + current_user=current_user(), + research_study_id=research_study_id, + patch_dstu2=True, + ignore_qb_requirement=True, + patient_ids=[patient_id] + ) + # filter qnr data to a manageable result data set + qnr_data = [] + for row in qnr_responses['entry']: + i = {} + d = row['resource'] + i['auth_method'] = d['encounter']['auth_method'] + i['encounter_period'] = d['encounter']['period'] + i['document_authored'] = d['authored'] + i['ae_session'] = d['identifier']['value'] + i['questionnaire'] = d['questionnaire']['reference'].split('/')[-1] + i['status'] = d['status'] + i['org'] = d['subject']['careProvider'][0]['display'] + i['visit'] = d['timepoint'] + qnr_data.append(i) + + kwargs = { + "rps": rps, + "status": status, + "posted": posted, + "timeline": results, + "adherence_data": adherence_data, + "qnr_data": qnr_data + } if trace: - return jsonify( - rps=rps, - status=status, - posted=posted, - timeline=results, - adherence_data=adherence_data, - trace=dump_trace("END time line lookup")) - return jsonify( - rps=rps, status=status, posted=posted, timeline=results, - adherence_data=adherence_data) + kwargs["trace"] = dump_trace("END time line lookup") + + return jsonify(**kwargs) @patient_api.route('/api/patient//timewarp/') From e210843b909e487c6cf5173d768ce0eb0b7bae01 Mon Sep 17 00:00:00 2001 From: Paul Bugni Date: Wed, 3 Jan 2024 14:40:41 -0800 Subject: [PATCH 08/12] add `only` parameter to /timeline for isolating results to one attribute --- portal/views/patient.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/portal/views/patient.py b/portal/views/patient.py index 3fced522a..dce3f0827 100644 --- a/portal/views/patient.py +++ b/portal/views/patient.py @@ -314,6 +314,7 @@ def patient_timeline(patient_id): :param research_study_id: set to alternative research study ID - default 0 :param trace: set 'true' to view detailed logs generated, works best in concert with purge + :param only: set to filter all results but top level attribute given """ from ..date_tools import FHIR_datetime, RelativeDelta @@ -500,6 +501,11 @@ def get_recur_id(qnr): if trace: kwargs["trace"] = dump_trace("END time line lookup") + only = request.args.get('only', False) + if only: + if only not in kwargs: + raise ValueError(f"{only} not in {kwargs.keys()}") + return jsonify(only, kwargs[only]) return jsonify(**kwargs) From 6d826837e9f94cd7ee0f086f743289ba03cdf6ee Mon Sep 17 00:00:00 2001 From: Paul Bugni Date: Wed, 3 Jan 2024 14:42:08 -0800 Subject: [PATCH 09/12] possible in unique cases, for `consent` to be undefined. --- portal/models/reporting.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/portal/models/reporting.py b/portal/models/reporting.py index 2566a4a06..9e45827cf 100644 --- a/portal/models/reporting.py +++ b/portal/models/reporting.py @@ -157,9 +157,12 @@ def empro_row_detail(row, ts_reporting): # Expired status ambiguous for EMPRO - either not available # due to complex business rules around start or walked off # the end. Assume if consent + 1year > now, it's the former. - consent = datetime.strptime(row["consent"], "%d-%b-%Y %H:%M:%S") - if consent + timedelta(days=365) > as_of_date: + if not row.get("consent"): row["status"] = "Not Yet Available" + else: + consent = datetime.strptime(row["consent"], "%d-%b-%Y %H:%M:%S") + if consent + timedelta(days=365) > as_of_date: + row["status"] = "Not Yet Available" if last_viable: general_row_detail(row, patient, last_viable) From 017933a5abfa2ddbdbbb39414a1f9eaaa89d9f1b Mon Sep 17 00:00:00 2001 From: Paul Bugni Date: Wed, 3 Jan 2024 15:37:53 -0800 Subject: [PATCH 10/12] as noticed in code review, the previous test for expired versus not yet available didn't consider delayed start timing. use the user's timeline, only consider it expired if 12th month visit expiration has passed. --- portal/models/reporting.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/portal/models/reporting.py b/portal/models/reporting.py index 9e45827cf..b41d46e2f 100644 --- a/portal/models/reporting.py +++ b/portal/models/reporting.py @@ -22,7 +22,7 @@ from .overall_status import OverallStatus from .questionnaire_response import aggregate_responses from .qb_status import QB_Status -from .qb_timeline import qb_status_visit_name +from .qb_timeline import QBT, qb_status_visit_name from .questionnaire_bank import visit_name from .questionnaire_response import ( QNR_results, @@ -154,15 +154,18 @@ def empro_row_detail(row, ts_reporting): row = patient_data(patient) row["status"] = status if status == "Expired" and research_study_id == EMPRO_RS_ID: - # Expired status ambiguous for EMPRO - either not available - # due to complex business rules around start or walked off - # the end. Assume if consent + 1year > now, it's the former. - if not row.get("consent"): + # Expired status ambiguous for EMPRO study. + # - If the last available questionnaire in the study is present in + # the user's timeline and the expired date has passed, it is + # legitimately "Expired". + # - Otherwise, due to complex business rules around delayed + # start/availability mark as "Not Yet Available" + exp_row = QBT.query.filter(QBT.research_study_id == EMPRO_RS_ID).filter( + QBT.user_id == patient.id).filter( + QBT.status == 'expired').filter( + QBT.qb_iteration == 10).first() # baseline is 1, 11 iterations base 0 + if not exp_row or exp_row.at > as_of_date: row["status"] = "Not Yet Available" - else: - consent = datetime.strptime(row["consent"], "%d-%b-%Y %H:%M:%S") - if consent + timedelta(days=365) > as_of_date: - row["status"] = "Not Yet Available" if last_viable: general_row_detail(row, patient, last_viable) From 0dbbe80530ecdc575643814fd49eb9de0db75a4e Mon Sep 17 00:00:00 2001 From: Paul Bugni Date: Wed, 3 Jan 2024 16:42:52 -0800 Subject: [PATCH 11/12] filter qnr_data in /timeline by research_study_id and add missing comment. --- portal/models/questionnaire_response.py | 2 ++ portal/views/patient.py | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/portal/models/questionnaire_response.py b/portal/models/questionnaire_response.py index 77ba003f5..3ca4981a8 100644 --- a/portal/models/questionnaire_response.py +++ b/portal/models/questionnaire_response.py @@ -854,6 +854,8 @@ def aggregate_responses( :param celery_task: if defined, send occasional progress updates :param patient_ids: if defined, limit result set to given patient list + NB: research_study_id not used to filter / restrict query set, but rather + for lookup of visit name. Use instrument_ids to restrict query set. """ from .qb_timeline import qb_status_visit_name # avoid cycle diff --git a/portal/views/patient.py b/portal/views/patient.py index dce3f0827..ac307f323 100644 --- a/portal/views/patient.py +++ b/portal/views/patient.py @@ -31,7 +31,11 @@ from ..models.questionnaire_response import QuestionnaireResponse from ..models.reference import Reference from ..models.reporting import single_patient_adherence_data -from ..models.research_study import EMPRO_RS_ID, ResearchStudy +from ..models.research_study import ( + EMPRO_RS_ID, + ResearchStudy, + research_study_id_from_questionnaire +) from ..models.role import ROLE from ..models.user import User, current_user, get_user from ..timeout_lock import ADHERENCE_DATA_KEY, CacheModeration @@ -480,11 +484,17 @@ def get_recur_id(qnr): for row in qnr_responses['entry']: i = {} d = row['resource'] + i['questionnaire'] = d['questionnaire']['reference'].split('/')[-1] + + # qnr_responses return all. filter to requested research_study + study_id = research_study_id_from_questionnaire(i['questionnaire']) + if study_id != research_study_id: + continue + i['auth_method'] = d['encounter']['auth_method'] i['encounter_period'] = d['encounter']['period'] i['document_authored'] = d['authored'] i['ae_session'] = d['identifier']['value'] - i['questionnaire'] = d['questionnaire']['reference'].split('/')[-1] i['status'] = d['status'] i['org'] = d['subject']['careProvider'][0]['display'] i['visit'] = d['timepoint'] From 933d633d0f705a8ae349714fe18fe75d37cfb387 Mon Sep 17 00:00:00 2001 From: Paul Bugni Date: Thu, 4 Jan 2024 17:07:18 -0800 Subject: [PATCH 12/12] Fix failing tests - can't put items in a cache for 1 day and then purge anything less than a day old, and expect them to still be in the cache. --- portal/models/reporting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portal/models/reporting.py b/portal/models/reporting.py index b41d46e2f..2b29b17ec 100644 --- a/portal/models/reporting.py +++ b/portal/models/reporting.py @@ -272,7 +272,7 @@ def cache_adherence_data( as_of_date = datetime.utcnow() # Purge any rows that have or will soon expire - valid = (as_of_date + timedelta(days=1)) + valid = (as_of_date + timedelta(hours=1)) AdherenceData.query.filter(AdherenceData.valid_till < valid).delete() db.session.commit()