From ed62a1d2cfb3cc6d173db4493ea311a92c01fc47 Mon Sep 17 00:00:00 2001 From: pbugni Date: Thu, 8 Aug 2024 17:43:25 -0700 Subject: [PATCH] additional database indexes as per TN-3308 (#4396) Using the table in [TN-3308](https://movember.atlassian.net/browse/TN-3308) as a guide, pencil traced the code to find the common queries on each respective table. I'd then assemble a query using `flask shell` to get the exact SQLAlchemy syntax, and then attempt before and after `explain $query` times, to confirm it would be of value. Note the few in the migration that are directly added via SQL `exec` as the SQLAlchemy syntax didn't support (at least I couldn't find a way). --------- Co-authored-by: Ivan Cvitkovic --- .../6120fcfc474a_add_missing_indexes.py | 67 +++++++++++++++++++ portal/models/audit.py | 6 +- portal/models/communication.py | 6 +- portal/models/encounter.py | 5 +- portal/models/message.py | 2 +- portal/models/observation.py | 6 +- portal/models/questionnaire_response.py | 3 +- portal/models/tou.py | 2 +- 8 files changed, 83 insertions(+), 14 deletions(-) create mode 100644 portal/migrations/versions/6120fcfc474a_add_missing_indexes.py diff --git a/portal/migrations/versions/6120fcfc474a_add_missing_indexes.py b/portal/migrations/versions/6120fcfc474a_add_missing_indexes.py new file mode 100644 index 0000000000..b9736fe9ca --- /dev/null +++ b/portal/migrations/versions/6120fcfc474a_add_missing_indexes.py @@ -0,0 +1,67 @@ +"""add missing indexes + +Revision ID: 6120fcfc474a +Revises: cf586ed4f043 +Create Date: 2024-08-06 17:49:02.447759 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '6120fcfc474a' +down_revision = 'cf586ed4f043' + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_index(op.f('ix_audit_context'), 'audit', ['context'], unique=False) + op.create_index(op.f('ix_audit_subject_id'), 'audit', ['subject_id'], unique=False) + op.create_index(op.f('ix_audit_user_id'), 'audit', ['user_id'], unique=False) + op.create_index(op.f('ix_communications_communication_request_id'), 'communications', ['communication_request_id'], unique=False) + op.create_index(op.f('ix_communications_status'), 'communications', ['status'], unique=False) + op.create_index(op.f('ix_communications_user_id'), 'communications', ['user_id'], unique=False) + op.create_index(op.f('ix_email_messages_recipients'), 'email_messages', ['recipients'], unique=False) + op.create_index(op.f('ix_encounter_codings_encounter_id'), 'encounter_codings', ['encounter_id'], unique=False) + op.create_index(op.f('ix_encounters_status'), 'encounters', ['status'], unique=False) + op.create_index(op.f('ix_encounters_user_id'), 'encounters', ['user_id'], unique=False) + op.create_index(op.f('ix_questionnaire_responses_encounter_id'), 'questionnaire_responses', ['encounter_id'], unique=False) + op.create_index(op.f('ix_questionnaire_responses_subject_id'), 'questionnaire_responses', ['subject_id'], unique=False) + op.create_index(op.f('ix_tou_audit_id'), 'tou', ['audit_id'], unique=False) + op.create_index(op.f('ix_user_observations_audit_id'), 'user_observations', ['audit_id'], unique=False) + op.create_index(op.f('ix_user_observations_observation_id'), 'user_observations', ['observation_id'], unique=False) + op.create_index(op.f('ix_user_observations_user_id'), 'user_observations', ['user_id'], unique=False) + +# The following are added by hand, as the syntax for JSONB indexes isn't available in active version + op.execute("CREATE INDEX IF NOT EXISTS idx_qnr_identifier_val2 ON questionnaire_responses(((document->'identifier')->'value'))") + op.execute("CREATE INDEX IF NOT EXISTS idx_qnr_identifier_sys2 ON questionnaire_responses(((document->'identifier')->'system'))") + op.execute("CREATE INDEX IF NOT EXISTS idx_qnr_authored ON questionnaire_responses((document->'authored'))") + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_tou_audit_id'), table_name='tou') + op.drop_index(op.f('ix_questionnaire_responses_subject_id'), table_name='questionnaire_responses') + op.drop_index(op.f('ix_questionnaire_responses_encounter_id'), table_name='questionnaire_responses') + op.drop_index(op.f('ix_encounters_user_id'), table_name='encounters') + op.drop_index(op.f('ix_encounters_status'), table_name='encounters') + op.drop_index(op.f('ix_encounter_codings_encounter_id'), table_name='encounter_codings') + op.drop_index(op.f('ix_email_messages_recipients'), table_name='email_messages') + op.drop_index(op.f('ix_communications_user_id'), table_name='communications') + op.drop_index(op.f('ix_communications_status'), table_name='communications') + op.drop_index(op.f('ix_communications_communication_request_id'), table_name='communications') + op.drop_index(op.f('ix_audit_user_id'), table_name='audit') + op.drop_index(op.f('ix_audit_subject_id'), table_name='audit') + op.drop_index(op.f('ix_audit_context'), table_name='audit') + op.drop_index(op.f('ix_user_observations_audit_id'), table_name='user_observations') + op.drop_index(op.f('ix_user_observations_observation_id'), table_name='user_observations') + op.drop_index(op.f('ix_user_observations_user_id'), table_name='user_observations') + + # and the ones added by hand + op.drop_index(op.f('idx_qnr_identifier_val2'), table_name='questionnaire_responses') + op.drop_index(op.f('idx_qnr_identifier_sys2'), table_name='questionnaire_responses') + op.drop_index(op.f('idx_qnr_authored'), table_name='questionnaire_responses') + # ### end Alembic commands ### diff --git a/portal/models/audit.py b/portal/models/audit.py index d7d5e47d9d..9759943cf3 100644 --- a/portal/models/audit.py +++ b/portal/models/audit.py @@ -29,9 +29,9 @@ class Audit(db.Model): """ id = db.Column(db.Integer, primary_key=True) - user_id = db.Column(db.ForeignKey('users.id'), nullable=False) - subject_id = db.Column(db.ForeignKey('users.id'), nullable=False) - _context = db.Column('context', db.Text, default='other', nullable=False) + user_id = db.Column(db.ForeignKey('users.id'), index=True, nullable=False) + subject_id = db.Column(db.ForeignKey('users.id'), index=True, nullable=False) + _context = db.Column('context', db.Text, default='other', index=True, nullable=False) timestamp = db.Column(db.DateTime, default=datetime.utcnow, nullable=False) version = db.Column(db.Text, default=lookup_version, nullable=False) comment = db.Column(db.Text) diff --git a/portal/models/communication.py b/portal/models/communication.py index 034d74f707..d6d59bd106 100644 --- a/portal/models/communication.py +++ b/portal/models/communication.py @@ -268,16 +268,16 @@ class Communication(db.Model): """ __tablename__ = 'communications' id = db.Column(db.Integer, primary_key=True) - status = db.Column('status', event_status_types, nullable=False) + status = db.Column('status', event_status_types, index=True, nullable=False) # FHIR Communication spec says `basedOn` can return to any # object. For current needs, this is always a CommunicationRequest communication_request_id = db.Column( - db.ForeignKey('communication_requests.id'), nullable=False) + db.ForeignKey('communication_requests.id'), index=True, nullable=False) communication_request = db.relationship('CommunicationRequest') user_id = db.Column(db.ForeignKey( - 'users.id', ondelete='cascade'), nullable=False) + 'users.id', ondelete='cascade'), index=True, nullable=False) # message_id is null until sent message_id = db.Column(db.ForeignKey( diff --git a/portal/models/encounter.py b/portal/models/encounter.py index 5ce5c6b326..3c30de40b8 100644 --- a/portal/models/encounter.py +++ b/portal/models/encounter.py @@ -21,7 +21,7 @@ class EncounterCodings(db.Model): __tablename__ = 'encounter_codings' id = db.Column(db.Integer, primary_key=True) - encounter_id = db.Column(db.ForeignKey('encounters.id'), nullable=False) + encounter_id = db.Column(db.ForeignKey('encounters.id'), index=True, nullable=False) coding_id = db.Column(db.ForeignKey('codings.id'), nullable=False) @@ -48,11 +48,12 @@ class Encounter(db.Model): """ __tablename__ = 'encounters' id = db.Column(db.Integer, primary_key=True) - status = db.Column('status', status_types, nullable=False) + status = db.Column('status', status_types, index=True, nullable=False) user_id = db.Column( db.ForeignKey( 'users.id', name='encounters_user_id_fk'), + index=True, nullable=False) start_time = db.Column(db.DateTime, nullable=False) """required whereas end_time is optional diff --git a/portal/models/message.py b/portal/models/message.py index 36209ca251..c78f59b927 100644 --- a/portal/models/message.py +++ b/portal/models/message.py @@ -98,7 +98,7 @@ class EmailMessage(db.Model): __tablename__ = 'email_messages' id = db.Column(db.Integer, primary_key=True) subject = db.Column(db.String(255), nullable=False) - _recipients = db.Column("recipients", db.Text, nullable=False) + _recipients = db.Column("recipients", db.Text, index=True, nullable=False) sender = db.Column(db.String(255), nullable=False) sent_at = db.Column(db.DateTime, default=datetime.utcnow) body = db.Column(db.Text, nullable=False) diff --git a/portal/models/observation.py b/portal/models/observation.py index b427fb4d0b..2022a5de46 100644 --- a/portal/models/observation.py +++ b/portal/models/observation.py @@ -137,13 +137,13 @@ class UserObservation(db.Model): __tablename__ = 'user_observations' id = db.Column(db.Integer, primary_key=True) user_id = db.Column(db.ForeignKey( - 'users.id', ondelete='CASCADE'), nullable=False) + 'users.id', ondelete='CASCADE'), index=True, nullable=False) observation_id = db.Column( - db.ForeignKey('observations.id'), nullable=False) + db.ForeignKey('observations.id'), index=True, nullable=False) encounter_id = db.Column(db.ForeignKey( 'encounters.id', name='user_observation_encounter_id_fk'), nullable=False) - audit_id = db.Column(db.ForeignKey('audit.id'), nullable=False) + audit_id = db.Column(db.ForeignKey('audit.id'), index=True, nullable=False) audit = db.relationship('Audit', cascade="save-update, delete") encounter = db.relationship('Encounter', cascade='delete') # There was a time when UserObservations were constrained to diff --git a/portal/models/questionnaire_response.py b/portal/models/questionnaire_response.py index faaa0a2f38..55e729b9b6 100644 --- a/portal/models/questionnaire_response.py +++ b/portal/models/questionnaire_response.py @@ -49,11 +49,12 @@ def default_status(context): __tablename__ = 'questionnaire_responses' id = db.Column(db.Integer, primary_key=True) - subject_id = db.Column(db.ForeignKey('users.id'), nullable=False) + subject_id = db.Column(db.ForeignKey('users.id'), index=True, nullable=False) subject = db.relationship("User", back_populates="questionnaire_responses") document = db.Column(JSONB) encounter_id = db.Column( db.ForeignKey('encounters.id', name='qr_encounter_id_fk'), + index=True, nullable=False) questionnaire_bank_id = db.Column( db.ForeignKey('questionnaire_banks.id'), nullable=True) diff --git a/portal/models/tou.py b/portal/models/tou.py index c68042e2a3..1578762cde 100644 --- a/portal/models/tou.py +++ b/portal/models/tou.py @@ -19,7 +19,7 @@ class ToU(db.Model): id = db.Column(db.Integer(), primary_key=True) agreement_url = db.Column( db.Text, server_default='predates agreement_url', nullable=False) - audit_id = db.Column(db.ForeignKey('audit.id'), nullable=False) + audit_id = db.Column(db.ForeignKey('audit.id'), index=True, nullable=False) organization_id = db.Column(db.ForeignKey('organizations.id')) type = db.Column('type', tou_types, nullable=False) active = db.Column(db.Boolean(), nullable=False, server_default='1')