From 73875fba310d31773e12997d3a381cc67521d11e Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Mon, 17 Jun 2024 15:33:57 -0400 Subject: [PATCH 01/67] show RNA type in sample hover --- seqr/views/apis/family_api.py | 17 +++++++++++++++-- ui/shared/components/panel/sample.jsx | 17 ++++++++++++----- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index 2c4ce793a0..4b78882950 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -23,7 +23,7 @@ from seqr.views.utils.project_context_utils import add_families_context, families_discovery_tags, add_project_tag_types, \ MME_TAG_NAME from seqr.models import Family, FamilyAnalysedBy, Individual, FamilyNote, Sample, VariantTag, AnalysisGroup, RnaSeqTpm, \ - PhenotypePrioritization, Project + PhenotypePrioritization, Project, RnaSeqOutlier, RnaSeqSpliceOutlier from seqr.views.utils.permissions_utils import check_project_permissions, get_project_and_check_pm_permissions, \ login_and_policies_required, user_is_analyst, has_case_review_permissions from seqr.views.utils.variant_utils import get_phenotype_prioritization, get_omim_intervals_query, DISCOVERY_CATEGORY @@ -45,9 +45,22 @@ def family_page_data(request, family_guid): sample_models = Sample.objects.filter(individual__family=family) samples = get_json_for_samples(sample_models, project_guid=project.guid, family_guid=family_guid, skip_nested=True, is_analyst=is_analyst) response = { - 'samplesByGuid': {s['sampleGuid']: s for s in samples}, + 'samplesByGuid': {s['sampleGuid']: {**s, 'rnaSeqTypes': []} for s in samples}, } + # Add Rna Seq metadata to samples + rna_seq_models = [ + (RnaSeqTpm, 'TPM'), + (RnaSeqOutlier, 'Outlier'), + (RnaSeqSpliceOutlier, 'Splice Outlier'), + ] + for model, rna_seq_type in rna_seq_models: + rna_seq_samples = model.objects.filter( + sample__in=sample_models, sample__is_active=True + ).values('sample__guid').distinct() + for rna_seq_sample in rna_seq_samples: + response['samplesByGuid'][rna_seq_sample['sample__guid']]['rnaSeqTypes'].append(rna_seq_type) + add_families_context(response, families, project.guid, request.user, is_analyst, has_case_review_perm) family_response = response['familiesByGuid'][family_guid] diff --git a/ui/shared/components/panel/sample.jsx b/ui/shared/components/panel/sample.jsx index 56bf048185..b09673e254 100644 --- a/ui/shared/components/panel/sample.jsx +++ b/ui/shared/components/panel/sample.jsx @@ -3,8 +3,8 @@ import PropTypes from 'prop-types' import styled from 'styled-components' import { Popup, Icon } from 'semantic-ui-react' -import { HorizontalSpacer } from '../Spacers' -import { DATASET_TYPE_SNV_INDEL_CALLS } from '../../utils/constants' +import { HorizontalSpacer, VerticalSpacer } from '../Spacers' +import { DATASET_TYPE_SNV_INDEL_CALLS, SAMPLE_TYPE_RNA } from '../../utils/constants' const Detail = styled.span` font-size: 11px; @@ -33,9 +33,16 @@ const Sample = React.memo(({ loadedSample, isOutdated, hoverDetails }) => ( } } - content={loadedSample ? - `data was${isOutdated ? ' previously ' : ''} ${hoverDetails ? `${hoverDetails} on ${new Date(loadedSample.loadedDate).toLocaleDateString()}` : 'loaded'}` : - 'no data available'} + content={ +
+ {loadedSample ? + `data was${isOutdated ? ' previously ' : ''} ${hoverDetails ? `${hoverDetails} on ${new Date(loadedSample.loadedDate).toLocaleDateString()}` : 'loaded'}` : + 'no data available'} + + {loadedSample?.sampleType && loadedSample.sampleType === SAMPLE_TYPE_RNA && + loadedSample?.rnaSeqTypes?.length > 0 && `RNAseq methods: ${loadedSample.rnaSeqTypes.join(', ')}`} +
+ } position="left center" /> )) From 9c6f2a688ac80ea96b8e35956d7026de106035cd Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Mon, 17 Jun 2024 16:59:26 -0400 Subject: [PATCH 02/67] remove hasRnaOutlierData --- seqr/views/apis/family_api.py | 5 ----- ui/pages/Project/components/FamilyTable/IndividualRow.jsx | 2 +- ui/shared/components/panel/sample.jsx | 4 ++-- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index 4b78882950..7182cb06b9 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -90,11 +90,6 @@ def family_page_data(request, family_guid): 'postDiscoveryOmimOptions': omim_map, }) - outlier_individual_guids = sample_models.filter(sample_type=Sample.SAMPLE_TYPE_RNA, is_active=True)\ - .exclude(rnaseqoutlier__isnull=True, rnaseqspliceoutlier__isnull=True).values_list('individual__guid', flat=True) - for individual_guid in outlier_individual_guids: - response['individualsByGuid'][individual_guid]['hasRnaOutlierData'] = True - tools_by_indiv = {} tools_agg = PhenotypePrioritization.objects.filter(individual__family=family).values('individual__guid').annotate( phenotypePrioritizationTools=ArrayAgg( diff --git a/ui/pages/Project/components/FamilyTable/IndividualRow.jsx b/ui/pages/Project/components/FamilyTable/IndividualRow.jsx index 4cb355b7c1..8717ebdde3 100644 --- a/ui/pages/Project/components/FamilyTable/IndividualRow.jsx +++ b/ui/pages/Project/components/FamilyTable/IndividualRow.jsx @@ -169,7 +169,7 @@ const DataDetails = React.memo(({ loadedSamples, individual, mmeSubmission }) => /> ) : )} - {individual.hasRnaOutlierData && ( + {loadedSamples.some(sample => sample.rnaSeqTypes?.length > 0) && (
( `data was${isOutdated ? ' previously ' : ''} ${hoverDetails ? `${hoverDetails} on ${new Date(loadedSample.loadedDate).toLocaleDateString()}` : 'loaded'}` : 'no data available'} - {loadedSample?.sampleType && loadedSample.sampleType === SAMPLE_TYPE_RNA && - loadedSample?.rnaSeqTypes?.length > 0 && `RNAseq methods: ${loadedSample.rnaSeqTypes.join(', ')}`} + {loadedSample.sampleType && loadedSample.sampleType === SAMPLE_TYPE_RNA && + loadedSample.rnaSeqTypes?.length > 0 && `RNAseq methods: ${loadedSample.rnaSeqTypes.join(', ')}`}
} position="left center" From d6dfb8f1cb32eb1e545191747024457301f25e96 Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Fri, 21 Jun 2024 12:58:02 -0400 Subject: [PATCH 03/67] additional values and post processing --- seqr/views/apis/family_api.py | 34 ++++++++++++++------------- seqr/views/utils/orm_to_json_utils.py | 5 ++-- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index 7182cb06b9..158957f70e 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -5,7 +5,7 @@ from collections import defaultdict from django.contrib.auth.models import User from django.contrib.postgres.aggregates import ArrayAgg -from django.db.models import Count, Q +from django.db.models import Count, Q, Case, When, Value, Exists, OuterRef from django.db.models.fields.files import ImageFieldFile from django.db.models.functions import JSONObject, Concat, Upper, Substr @@ -43,23 +43,25 @@ def family_page_data(request, family_guid): has_case_review_perm = has_case_review_permissions(project, request.user) sample_models = Sample.objects.filter(individual__family=family) - samples = get_json_for_samples(sample_models, project_guid=project.guid, family_guid=family_guid, skip_nested=True, is_analyst=is_analyst) - response = { - 'samplesByGuid': {s['sampleGuid']: {**s, 'rnaSeqTypes': []} for s in samples}, + additional_values = { + 'rnaSeqTpm': Case(When(Exists(RnaSeqTpm.objects.filter(sample_id=OuterRef('pk'))), then=Value('TPM')), default=None), + 'rnaSeqOutlier': Case(When(Exists(RnaSeqOutlier.objects.filter(sample_id=OuterRef('pk'))), then=Value('Outlier')), default=None), + 'rnaSeqSpliceOutlier': Case(When(Exists(RnaSeqSpliceOutlier.objects.filter(sample_id=OuterRef('pk'))), then=Value('Splice Outlier')), default=None), } + samples = get_json_for_samples( + sample_models, project_guid=project.guid, family_guid=family_guid, skip_nested=True, is_analyst=is_analyst, + additional_values=additional_values + ) + samples_by_guid = {} + for sample in samples: + tpm, outlier, splice_outlier = sample.pop('rnaSeqTpm'), sample.pop('rnaSeqOutlier'), sample.pop('rnaSeqSpliceOutlier') + if sample['sampleType'] == 'RNA': + sample['rnaSeqTypes'] = [value for value in [tpm, outlier, splice_outlier] if value] + samples_by_guid[sample['sampleGuid']] = sample - # Add Rna Seq metadata to samples - rna_seq_models = [ - (RnaSeqTpm, 'TPM'), - (RnaSeqOutlier, 'Outlier'), - (RnaSeqSpliceOutlier, 'Splice Outlier'), - ] - for model, rna_seq_type in rna_seq_models: - rna_seq_samples = model.objects.filter( - sample__in=sample_models, sample__is_active=True - ).values('sample__guid').distinct() - for rna_seq_sample in rna_seq_samples: - response['samplesByGuid'][rna_seq_sample['sample__guid']]['rnaSeqTypes'].append(rna_seq_type) + response = { + 'samplesByGuid': {s['sampleGuid']: s for s in samples}, + } add_families_context(response, families, project.guid, request.user, is_analyst, has_case_review_perm) family_response = response['familiesByGuid'][family_guid] diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index 67a943fc8f..8939c99d4b 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -346,16 +346,17 @@ def _get_sample_json_kwargs(project_guid=None, family_guid=None, individual_guid return {'guid_key': 'sampleGuid', **additional_kwargs, **kwargs} -def get_json_for_samples(samples, **kwargs): +def get_json_for_samples(samples, additional_values=None, **kwargs): """Returns a JSON representation of the given list of Samples. Args: samples (array): array of django models for the Samples. + additional_values (dict): additional values to include in the json Returns: array: array of json objects """ - return get_json_for_queryset(samples, **_get_sample_json_kwargs(**kwargs)) + return get_json_for_queryset(samples, additional_values=additional_values, **_get_sample_json_kwargs(**kwargs)) def get_json_for_sample(sample, **kwargs): From 88c86c5622952a43f0f133aa165ade5e866ad6fd Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Fri, 21 Jun 2024 14:04:48 -0400 Subject: [PATCH 04/67] extra --- seqr/views/apis/family_api.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index 158957f70e..8711601d00 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -4,8 +4,9 @@ import json from collections import defaultdict from django.contrib.auth.models import User -from django.contrib.postgres.aggregates import ArrayAgg -from django.db.models import Count, Q, Case, When, Value, Exists, OuterRef +from django.contrib.postgres.aggregates import ArrayAgg, JSONBAgg, StringAgg +from django.contrib.postgres.fields import ArrayField +from django.db.models import Count, Q, Case, When, Value, CharField, F, Exists, OuterRef from django.db.models.fields.files import ImageFieldFile from django.db.models.functions import JSONObject, Concat, Upper, Substr @@ -52,12 +53,10 @@ def family_page_data(request, family_guid): sample_models, project_guid=project.guid, family_guid=family_guid, skip_nested=True, is_analyst=is_analyst, additional_values=additional_values ) - samples_by_guid = {} for sample in samples: tpm, outlier, splice_outlier = sample.pop('rnaSeqTpm'), sample.pop('rnaSeqOutlier'), sample.pop('rnaSeqSpliceOutlier') if sample['sampleType'] == 'RNA': sample['rnaSeqTypes'] = [value for value in [tpm, outlier, splice_outlier] if value] - samples_by_guid[sample['sampleGuid']] = sample response = { 'samplesByGuid': {s['sampleGuid']: s for s in samples}, From c73e3f684999cff53c6d54298958f4ce4c1a6c38 Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Fri, 21 Jun 2024 14:20:56 -0400 Subject: [PATCH 05/67] test --- seqr/views/apis/family_api_tests.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py index eb22e8e85d..1664d72da6 100644 --- a/seqr/views/apis/family_api_tests.py +++ b/seqr/views/apis/family_api_tests.py @@ -31,6 +31,7 @@ INDIVIDUAL3_GUID = 'I000003_na19679' INDIVIDUAL_GUIDS = [INDIVIDUAL_GUID, INDIVIDUAL2_GUID, INDIVIDUAL3_GUID] +SAMPLE_GUIDS = ['S000129_na19675', 'S000130_na19678', 'S000131_na19679', 'S000151_na19675_1', 'S000152_na19675_d2', 'S000153_na19679'] class FamilyAPITest(AuthenticationTestCase): fixtures = ['users', '1kg_project', 'reference_data'] @@ -69,8 +70,7 @@ def test_family_page_data(self): self.assertEqual(len(response_json['individualsByGuid']), 3) individual = response_json['individualsByGuid'][INDIVIDUAL_GUID] - individual_fields = {'sampleGuids', 'igvSampleGuids', 'mmeSubmissionGuid', 'hasRnaOutlierData', - 'phenotypePrioritizationTools'} + individual_fields = {'sampleGuids', 'igvSampleGuids', 'mmeSubmissionGuid', 'phenotypePrioritizationTools'} individual_fields.update(INDIVIDUAL_FIELDS) self.assertSetEqual(set(individual.keys()), individual_fields) self.assertListEqual([ @@ -83,10 +83,6 @@ def test_family_page_data(self): ], [response_json['individualsByGuid'][guid].get('phenotypePrioritizationTools') for guid in INDIVIDUAL_GUIDS] ) - self.assertListEqual( - [True, False, True], - [response_json['individualsByGuid'][guid].get('hasRnaOutlierData', False) for guid in INDIVIDUAL_GUIDS] - ) self.assertSetEqual({PROJECT_GUID}, {i['projectGuid'] for i in response_json['individualsByGuid'].values()}) self.assertSetEqual({FAMILY_GUID}, {i['familyGuid'] for i in response_json['individualsByGuid'].values()}) @@ -96,6 +92,10 @@ def test_family_page_data(self): self.assertSetEqual({FAMILY_GUID}, {s['familyGuid'] for s in response_json['samplesByGuid'].values()}) self.assertEqual(len(individual['sampleGuids']), 3) self.assertTrue(set(individual['sampleGuids']).issubset(set(response_json['samplesByGuid'].keys()))) + self.assertListEqual( + [[], [], [], ['TPM', 'Splice Outlier'], ['TPM', 'Outlier', 'Splice Outlier'], ['Splice Outlier']], + [response_json['samplesByGuid'][guid].get('rnaSeqTypes', []) for guid in SAMPLE_GUIDS] + ) self.assertEqual(len(response_json['igvSamplesByGuid']), 1) self.assertSetEqual(set(next(iter(response_json['igvSamplesByGuid'].values())).keys()), IGV_SAMPLE_FIELDS) From a539726c4b8a49f3e8caea78819e5b5940c81462 Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Fri, 21 Jun 2024 14:26:18 -0400 Subject: [PATCH 06/67] unused --- seqr/views/apis/family_api.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index 8711601d00..5fe95572ac 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -4,9 +4,8 @@ import json from collections import defaultdict from django.contrib.auth.models import User -from django.contrib.postgres.aggregates import ArrayAgg, JSONBAgg, StringAgg -from django.contrib.postgres.fields import ArrayField -from django.db.models import Count, Q, Case, When, Value, CharField, F, Exists, OuterRef +from django.contrib.postgres.aggregates import ArrayAgg +from django.db.models import Count, Q, Case, When, Value, Exists, OuterRef from django.db.models.fields.files import ImageFieldFile from django.db.models.functions import JSONObject, Concat, Upper, Substr From e2d52f1276a3fba40493e5bafb1ebd78a4bd1762 Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Mon, 24 Jun 2024 13:57:22 -0400 Subject: [PATCH 07/67] with isnull --- seqr/views/apis/family_api.py | 13 +++++-------- seqr/views/apis/family_api_tests.py | 12 +++++++++--- seqr/views/utils/test_utils.py | 2 +- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index 5fe95572ac..50b294dc88 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -44,19 +44,16 @@ def family_page_data(request, family_guid): sample_models = Sample.objects.filter(individual__family=family) additional_values = { - 'rnaSeqTpm': Case(When(Exists(RnaSeqTpm.objects.filter(sample_id=OuterRef('pk'))), then=Value('TPM')), default=None), - 'rnaSeqOutlier': Case(When(Exists(RnaSeqOutlier.objects.filter(sample_id=OuterRef('pk'))), then=Value('Outlier')), default=None), - 'rnaSeqSpliceOutlier': Case(When(Exists(RnaSeqSpliceOutlier.objects.filter(sample_id=OuterRef('pk'))), then=Value('Splice Outlier')), default=None), + 'rnaSeqTypes': JSONObject( + hasRnaSeqTpm=Case(When(rnaseqtpm__isnull=False, then=True), default=False), + hasRnaSeqOutlier=Case(When(rnaseqoutlier__isnull=False, then=True), default=False), + hasRnaSeqSpliceOutlier=Case(When(rnaseqspliceoutlier__isnull=False, then=True), default=False), + ) } samples = get_json_for_samples( sample_models, project_guid=project.guid, family_guid=family_guid, skip_nested=True, is_analyst=is_analyst, additional_values=additional_values ) - for sample in samples: - tpm, outlier, splice_outlier = sample.pop('rnaSeqTpm'), sample.pop('rnaSeqOutlier'), sample.pop('rnaSeqSpliceOutlier') - if sample['sampleType'] == 'RNA': - sample['rnaSeqTypes'] = [value for value in [tpm, outlier, splice_outlier] if value] - response = { 'samplesByGuid': {s['sampleGuid']: s for s in samples}, } diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py index 1664d72da6..d585018b1c 100644 --- a/seqr/views/apis/family_api_tests.py +++ b/seqr/views/apis/family_api_tests.py @@ -92,9 +92,15 @@ def test_family_page_data(self): self.assertSetEqual({FAMILY_GUID}, {s['familyGuid'] for s in response_json['samplesByGuid'].values()}) self.assertEqual(len(individual['sampleGuids']), 3) self.assertTrue(set(individual['sampleGuids']).issubset(set(response_json['samplesByGuid'].keys()))) - self.assertListEqual( - [[], [], [], ['TPM', 'Splice Outlier'], ['TPM', 'Outlier', 'Splice Outlier'], ['Splice Outlier']], - [response_json['samplesByGuid'][guid].get('rnaSeqTypes', []) for guid in SAMPLE_GUIDS] + expected_rna_seq_types = [ + {'hasRnaSeqTpm': False, 'hasRnaSeqOutlier': False, 'hasRnaSeqSpliceOutlier': False} for _ in range(3) + ] + [ + {'hasRnaSeqTpm': True, 'hasRnaSeqOutlier': False, 'hasRnaSeqSpliceOutlier': True}, + {'hasRnaSeqTpm': True, 'hasRnaSeqOutlier': True, 'hasRnaSeqSpliceOutlier': True}, + {'hasRnaSeqTpm': False, 'hasRnaSeqOutlier': False, 'hasRnaSeqSpliceOutlier': True}, + ] + self.assertListEqual(expected_rna_seq_types, + [response_json['samplesByGuid'][guid]['rnaSeqTypes'] for guid in SAMPLE_GUIDS] ) self.assertEqual(len(response_json['igvSamplesByGuid']), 1) diff --git a/seqr/views/utils/test_utils.py b/seqr/views/utils/test_utils.py index f0ed234d4f..bb0d509835 100644 --- a/seqr/views/utils/test_utils.py +++ b/seqr/views/utils/test_utils.py @@ -793,7 +793,7 @@ def _get_list_param(call, param): SAMPLE_FIELDS = { 'projectGuid', 'familyGuid', 'individualGuid', 'sampleGuid', 'createdDate', 'sampleType', 'sampleId', 'isActive', - 'loadedDate', 'datasetType', 'elasticsearchIndex', + 'loadedDate', 'datasetType', 'elasticsearchIndex', 'rnaSeqTypes' } IGV_SAMPLE_FIELDS = { From 6268b1041b24107a25359bbd169f5ad8bc6b49a1 Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Mon, 24 Jun 2024 13:58:04 -0400 Subject: [PATCH 08/67] with outerref --- seqr/views/apis/family_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index 50b294dc88..1987a849a8 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -45,9 +45,9 @@ def family_page_data(request, family_guid): sample_models = Sample.objects.filter(individual__family=family) additional_values = { 'rnaSeqTypes': JSONObject( - hasRnaSeqTpm=Case(When(rnaseqtpm__isnull=False, then=True), default=False), - hasRnaSeqOutlier=Case(When(rnaseqoutlier__isnull=False, then=True), default=False), - hasRnaSeqSpliceOutlier=Case(When(rnaseqspliceoutlier__isnull=False, then=True), default=False), + hasRnaSeqTpm=Case(When(Exists(RnaSeqTpm.objects.filter(sample_id=OuterRef('pk'))), then=True), default=False), + hasRnaSeqOutlier=Case(When(Exists(RnaSeqOutlier.objects.filter(sample_id=OuterRef('pk'))), then=True), default=False), + hasRnaSeqSpliceOutlier=Case(When(Exists(RnaSeqSpliceOutlier.objects.filter(sample_id=OuterRef('pk'))), then=True), default=False), ) } samples = get_json_for_samples( From 50213d9d89841c77fbf8d0c6ff82270d2e348426 Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Tue, 25 Jun 2024 13:33:15 -0400 Subject: [PATCH 09/67] changes --- seqr/views/apis/family_api.py | 21 +++++++++++---------- seqr/views/apis/family_api_tests.py | 12 +++--------- seqr/views/utils/orm_to_json_utils.py | 4 ++-- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index 1987a849a8..4e862174fb 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -43,20 +43,21 @@ def family_page_data(request, family_guid): has_case_review_perm = has_case_review_permissions(project, request.user) sample_models = Sample.objects.filter(individual__family=family) - additional_values = { - 'rnaSeqTypes': JSONObject( - hasRnaSeqTpm=Case(When(Exists(RnaSeqTpm.objects.filter(sample_id=OuterRef('pk'))), then=True), default=False), - hasRnaSeqOutlier=Case(When(Exists(RnaSeqOutlier.objects.filter(sample_id=OuterRef('pk'))), then=True), default=False), - hasRnaSeqSpliceOutlier=Case(When(Exists(RnaSeqSpliceOutlier.objects.filter(sample_id=OuterRef('pk'))), then=True), default=False), - ) - } samples = get_json_for_samples( - sample_models, project_guid=project.guid, family_guid=family_guid, skip_nested=True, is_analyst=is_analyst, - additional_values=additional_values + sample_models, project_guid=project.guid, family_guid=family_guid, skip_nested=True, is_analyst=is_analyst ) response = { - 'samplesByGuid': {s['sampleGuid']: s for s in samples}, + 'samplesByGuid': {s['sampleGuid']: s for s in samples} + } + rna_type_samples = { + 'TPM': set(RnaSeqTpm.objects.filter(sample__in=sample_models).values_list('sample__guid', flat=True)), + 'Expression Outlier': set(RnaSeqOutlier.objects.filter(sample__in=sample_models).values_list('sample__guid', flat=True)), + 'Splice Outlier': set(RnaSeqSpliceOutlier.objects.filter(sample__in=sample_models).values_list('sample__guid', flat=True)), } + for sample in response['samplesByGuid'].values(): + sample['rnaSeqTypes'] = [ + rnaseq_type for rnaseq_type, sample_ids in rna_type_samples.items() if sample['sampleGuid'] in sample_ids + ] add_families_context(response, families, project.guid, request.user, is_analyst, has_case_review_perm) family_response = response['familiesByGuid'][family_guid] diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py index d585018b1c..cbbcb8f842 100644 --- a/seqr/views/apis/family_api_tests.py +++ b/seqr/views/apis/family_api_tests.py @@ -92,15 +92,9 @@ def test_family_page_data(self): self.assertSetEqual({FAMILY_GUID}, {s['familyGuid'] for s in response_json['samplesByGuid'].values()}) self.assertEqual(len(individual['sampleGuids']), 3) self.assertTrue(set(individual['sampleGuids']).issubset(set(response_json['samplesByGuid'].keys()))) - expected_rna_seq_types = [ - {'hasRnaSeqTpm': False, 'hasRnaSeqOutlier': False, 'hasRnaSeqSpliceOutlier': False} for _ in range(3) - ] + [ - {'hasRnaSeqTpm': True, 'hasRnaSeqOutlier': False, 'hasRnaSeqSpliceOutlier': True}, - {'hasRnaSeqTpm': True, 'hasRnaSeqOutlier': True, 'hasRnaSeqSpliceOutlier': True}, - {'hasRnaSeqTpm': False, 'hasRnaSeqOutlier': False, 'hasRnaSeqSpliceOutlier': True}, - ] - self.assertListEqual(expected_rna_seq_types, - [response_json['samplesByGuid'][guid]['rnaSeqTypes'] for guid in SAMPLE_GUIDS] + self.assertListEqual( + [[], [], [], ['TPM', 'Splice Outlier'], ['TPM', 'Expression Outlier', 'Splice Outlier'], ['Splice Outlier']], + [response_json['samplesByGuid'][guid].get('rnaSeqTypes', {}) for guid in SAMPLE_GUIDS] ) self.assertEqual(len(response_json['igvSamplesByGuid']), 1) diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index 8939c99d4b..e14ad01066 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -346,7 +346,7 @@ def _get_sample_json_kwargs(project_guid=None, family_guid=None, individual_guid return {'guid_key': 'sampleGuid', **additional_kwargs, **kwargs} -def get_json_for_samples(samples, additional_values=None, **kwargs): +def get_json_for_samples(samples, **kwargs): """Returns a JSON representation of the given list of Samples. Args: @@ -356,7 +356,7 @@ def get_json_for_samples(samples, additional_values=None, **kwargs): array: array of json objects """ - return get_json_for_queryset(samples, additional_values=additional_values, **_get_sample_json_kwargs(**kwargs)) + return get_json_for_queryset(samples, **_get_sample_json_kwargs(**kwargs)) def get_json_for_sample(sample, **kwargs): From 6602382908760227c1ebe636b228a7b5425119e2 Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Tue, 25 Jun 2024 13:34:59 -0400 Subject: [PATCH 10/67] lint stuff --- seqr/views/apis/family_api.py | 2 +- seqr/views/utils/orm_to_json_utils.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index 4e862174fb..dcae71adeb 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -5,7 +5,7 @@ from collections import defaultdict from django.contrib.auth.models import User from django.contrib.postgres.aggregates import ArrayAgg -from django.db.models import Count, Q, Case, When, Value, Exists, OuterRef +from django.db.models import Count, Q from django.db.models.fields.files import ImageFieldFile from django.db.models.functions import JSONObject, Concat, Upper, Substr diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index e14ad01066..67a943fc8f 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -351,7 +351,6 @@ def get_json_for_samples(samples, **kwargs): Args: samples (array): array of django models for the Samples. - additional_values (dict): additional values to include in the json Returns: array: array of json objects """ From a28272a935f32368f887a0eab848a37aabdd3a93 Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Tue, 25 Jun 2024 13:41:18 -0400 Subject: [PATCH 11/67] key from project api tests? --- seqr/views/utils/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/views/utils/test_utils.py b/seqr/views/utils/test_utils.py index bb0d509835..f0ed234d4f 100644 --- a/seqr/views/utils/test_utils.py +++ b/seqr/views/utils/test_utils.py @@ -793,7 +793,7 @@ def _get_list_param(call, param): SAMPLE_FIELDS = { 'projectGuid', 'familyGuid', 'individualGuid', 'sampleGuid', 'createdDate', 'sampleType', 'sampleId', 'isActive', - 'loadedDate', 'datasetType', 'elasticsearchIndex', 'rnaSeqTypes' + 'loadedDate', 'datasetType', 'elasticsearchIndex', } IGV_SAMPLE_FIELDS = { From 41ffb9b2e901b7f454a9c768e7a9933345bb99c3 Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Tue, 25 Jun 2024 13:52:35 -0400 Subject: [PATCH 12/67] fix the last test --- seqr/views/apis/family_api_tests.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py index cbbcb8f842..bd43d8d697 100644 --- a/seqr/views/apis/family_api_tests.py +++ b/seqr/views/apis/family_api_tests.py @@ -31,8 +31,11 @@ INDIVIDUAL3_GUID = 'I000003_na19679' INDIVIDUAL_GUIDS = [INDIVIDUAL_GUID, INDIVIDUAL2_GUID, INDIVIDUAL3_GUID] + +FAMILY_API_SAMPLE_FIELDS = {*SAMPLE_FIELDS, 'rnaSeqTypes'} SAMPLE_GUIDS = ['S000129_na19675', 'S000130_na19678', 'S000131_na19679', 'S000151_na19675_1', 'S000152_na19675_d2', 'S000153_na19679'] + class FamilyAPITest(AuthenticationTestCase): fixtures = ['users', '1kg_project', 'reference_data'] @@ -87,7 +90,7 @@ def test_family_page_data(self): self.assertSetEqual({FAMILY_GUID}, {i['familyGuid'] for i in response_json['individualsByGuid'].values()}) self.assertEqual(len(response_json['samplesByGuid']), 6) - self.assertSetEqual(set(next(iter(response_json['samplesByGuid'].values())).keys()), SAMPLE_FIELDS) + self.assertSetEqual(set(next(iter(response_json['samplesByGuid'].values())).keys()), FAMILY_API_SAMPLE_FIELDS) self.assertSetEqual({PROJECT_GUID}, {s['projectGuid'] for s in response_json['samplesByGuid'].values()}) self.assertSetEqual({FAMILY_GUID}, {s['familyGuid'] for s in response_json['samplesByGuid'].values()}) self.assertEqual(len(individual['sampleGuids']), 3) From a4058264224e624631390698c4d2fbda9dfe60c1 Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Wed, 26 Jun 2024 13:36:43 -0400 Subject: [PATCH 13/67] address deprecation warning for url() --- seqr/urls.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/seqr/urls.py b/seqr/urls.py index 82e6b91763..2cdce9c593 100644 --- a/seqr/urls.py +++ b/seqr/urls.py @@ -8,7 +8,8 @@ from seqr.views.apis.dataset_api import add_variants_dataset_handler from settings import ENABLE_DJANGO_DEBUG_TOOLBAR, MEDIA_ROOT, API_LOGIN_REQUIRED_URL, LOGIN_URL, DEBUG, \ API_POLICY_REQUIRED_URL -from django.conf.urls import url, include +from django.conf.urls import include +from django.urls import re_path, path from django.contrib import admin from django.views.generic.base import RedirectView import django.views.static @@ -358,25 +359,26 @@ 'matchmaker/v1/metrics': external_api.mme_metrics_proxy, } -urlpatterns = [url('^status', status_view)] +urlpatterns = [path('status', status_view)] # anvil workspace anvil_workspace_url = 'workspace/(?P[^/]+)/(?P[^/]+)' -urlpatterns += [url("^%(anvil_workspace_url)s$" % locals(), anvil_workspace_page)] +urlpatterns += [re_path(r"^%(anvil_workspace_url)s$" % locals(), anvil_workspace_page)] # core react page templates -urlpatterns += [url("^%(url_endpoint)s$" % locals(), main_app) for url_endpoint in react_app_pages] -urlpatterns += [url("^%(url_endpoint)s$" % locals(), no_login_main_app) for url_endpoint in no_login_react_app_pages] +urlpatterns += [re_path(r"^%(url_endpoint)s$" % locals(), main_app) for url_endpoint in react_app_pages] +urlpatterns += [re_path(r"^%(url_endpoint)s$" % locals(), no_login_main_app) for url_endpoint in no_login_react_app_pages] # api for url_endpoint, handler_function in api_endpoints.items(): - urlpatterns.append( url("^api/%(url_endpoint)s$" % locals(), handler_function) ) + urlpatterns.append(re_path(r"^api/%(url_endpoint)s$" % locals(), handler_function)) + # login/ logout urlpatterns += [ - url('^logout$', logout_view), - url(API_LOGIN_REQUIRED_URL.lstrip('/'), login_required_error), - url(API_POLICY_REQUIRED_URL.lstrip('/'), policies_required_error), + path('logout', logout_view), + path(API_LOGIN_REQUIRED_URL.lstrip('/'), login_required_error), + path(API_POLICY_REQUIRED_URL.lstrip('/'), policies_required_error), ] handler401 = 'seqr.views.apis.auth_api.app_login_required_error' @@ -389,12 +391,12 @@ ])) urlpatterns += [ - url(kibana_urls, proxy_to_kibana, name='proxy_to_kibana'), + re_path(kibana_urls, proxy_to_kibana, name='proxy_to_kibana'), ] urlpatterns += [ - url(r'^admin/login/$', RedirectView.as_view(url=LOGIN_URL, permanent=True, query_string=True)), - url(r'^admin/', admin.site.urls), + re_path(r'^admin/login/$', RedirectView.as_view(url=LOGIN_URL, permanent=True, query_string=True)), + re_path(r'^admin/', admin.site.urls), ] # The /media urlpattern is not needed if we are storing static media in a GCS bucket, @@ -402,23 +404,23 @@ # instead, set MEDIA_ROOT in settings.py to that local path, and then this urlpattern will be enabled. if MEDIA_ROOT: urlpatterns += [ - url(r'^media/(?P.*)$', django.views.static.serve, { + re_path(r'^media/(?P.*)$', django.views.static.serve, { 'document_root': MEDIA_ROOT, }), ] urlpatterns += [ - url('', include('social_django.urls')), + path('', include('social_django.urls')), ] if DEBUG: urlpatterns += [ - url(r'^hijack/', include('hijack.urls')), + re_path(r'^hijack/', include('hijack.urls')), ] # django debug toolbar if ENABLE_DJANGO_DEBUG_TOOLBAR: import debug_toolbar urlpatterns = [ - url(r'^__debug__/', include(debug_toolbar.urls)), + re_path(r'^__debug__/', include(debug_toolbar.urls)), ] + urlpatterns From dbdb0d93d32afdb90d72d1886a021b0b9784dd1f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 26 Jun 2024 17:01:51 -0400 Subject: [PATCH 14/67] document extra response keys --- seqr/views/apis/report_api_tests.py | 16 ++++++++-------- seqr/views/apis/summary_data_api_tests.py | 12 ++++++------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index 00ade3156d..222182fd1d 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -496,12 +496,12 @@ 'MME': False, 'additional_family_members_with_variant': '', 'allele_balance_or_heteroplasmy_percentage': None, - 'analysisStatus': 'Q', - 'analysis_groups': '', + 'analysisStatus': 'Q', # unused + 'analysis_groups': '', # unused 'clinvar': None, 'condition_id': None, - 'consanguinity': 'Unknown', - 'end': None, + 'consanguinity': 'Unknown', # unused + 'end': None, # unused 'hgvsc': '', 'hgvsp': '', 'method_of_discovery': 'SR-ES', @@ -509,7 +509,7 @@ 'phenotype_contribution': 'Full', 'partial_contribution_explained': '', 'seqr_chosen_consequence': None, - 'svName': None, + 'svName': None, # unused 'svType': None, 'sv_name': None, 'transcript': None, @@ -1178,7 +1178,7 @@ def test_family_metadata(self): 'solve_status': 'Partially solved', 'actual_inheritance': 'unknown', 'condition_id': 'OMIM:616126', - 'condition_inheritance': 'Autosomal recessive', + 'condition_inheritance': 'Autosomal recessive', # unused 'known_condition_name': 'Immunodeficiency 38', 'date_data_generation': '2017-02-05', 'data_type': 'WES', @@ -1266,7 +1266,7 @@ def test_variant_metadata(self): 'familyGuid': 'F000002_2', 'family_id': '2', 'gene_of_interest': 'RP11', - 'gene_id': 'ENSG00000135953', + 'gene_id': 'ENSG00000135953', # unused 'gene_known_for_phenotype': 'Known', 'genetic_findings_id': 'HG00731_1_248367227', 'known_condition_name': 'mitochondrial disease', @@ -1331,7 +1331,7 @@ def test_variant_metadata(self): 'displayName': '12', 'familyGuid': 'F000012_12', 'family_id': '12', - 'family_history': 'Yes', + 'family_history': 'Yes', # unused 'gene_of_interest': 'OR4G11P', 'gene_id': 'ENSG00000240361', 'gene_known_for_phenotype': 'Candidate', diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index 8785e361e0..b5acdf21cc 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -36,7 +36,7 @@ "projectGuid": "R0003_test", "num_saved_variants": 2, "solve_status": "Partially solved", - "sample_id": "NA20889", + "sample_id": "NA20889", # unused "gene_known_for_phenotype-1": "Candidate", "gene_known_for_phenotype-2": "Candidate", "variant_inheritance-1": "unknown", @@ -52,7 +52,7 @@ "sv_name-2": "DEL:chr1:249045487-249045898", "chrom-2": "1", "pos-2": 249045487, - 'end-2': 249045898, + 'end-2': 249045898, # unused "maternal_id": "", "paternal_id": "", "maternal_guid": "", @@ -71,7 +71,7 @@ "chrom-1": "1", "alt-1": "T", "gene_of_interest-1": "OR4G11P", - "gene_id-1": "ENSG00000240361", + "gene_id-1": "ENSG00000240361", # unused 'variant_reference_assembly-1': 'GRCh37', 'variant_reference_assembly-2': 'GRCh37', "pmid_id": None, @@ -96,16 +96,16 @@ 'seqr_chosen_consequence-2': None, 'gene_of_interest-2': None, 'gene_id-2': None, - 'svName-2': None, + 'svName-2': None, # unused 'svType-1': None, 'sv_name-1': None, 'svName-1': None, 'end-1': None, - 'allele_balance_or_heteroplasmy_percentage-1': None, + 'allele_balance_or_heteroplasmy_percentage-1': None, # unused 'allele_balance_or_heteroplasmy_percentage-2': None, 'notes-1': None, 'notes-2': None, - 'tags-1': ['Tier 1 - Novel gene and phenotype'], + 'tags-1': ['Tier 1 - Novel gene and phenotype'], # unused 'tags-2': ['Tier 1 - Novel gene and phenotype'], 'phenotype_contribution-1': 'Partial', 'phenotype_contribution-2': 'Full', From 360ed6182f7476fe4f5ace394583cceaf384af32 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 26 Jun 2024 17:21:03 -0400 Subject: [PATCH 15/67] share metadata columsn for individual and family metadata --- seqr/views/apis/report_api_tests.py | 2 ++ ui/pages/Report/components/FamilyMetadata.jsx | 17 ++--------------- .../components/IndividualMetadata.jsx | 17 ++--------------- ui/shared/components/table/LoadReportTable.jsx | 2 +- ui/shared/utils/constants.js | 17 +++++++++++++++++ 5 files changed, 24 insertions(+), 31 deletions(-) diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index 222182fd1d..6ff66cbaf4 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -513,6 +513,7 @@ 'svType': None, 'sv_name': None, 'transcript': None, + # TODO missing internal_project_id } PARTICIPANT_TABLE = [ @@ -1195,6 +1196,7 @@ def test_family_metadata(self): 'analysisStatus': 'Q', 'analysis_groups': '', 'consanguinity': 'Unknown', + # TODO missing internal_project_id }) # Test all projects diff --git a/ui/pages/Report/components/FamilyMetadata.jsx b/ui/pages/Report/components/FamilyMetadata.jsx index 412dc55848..93bfde2892 100644 --- a/ui/pages/Report/components/FamilyMetadata.jsx +++ b/ui/pages/Report/components/FamilyMetadata.jsx @@ -1,33 +1,20 @@ import React from 'react' import LoadReportTable from 'shared/components/table/LoadReportTable' -import { FAMILY_ANALYSIS_STATUS_LOOKUP } from 'shared/utils/constants' +import { BASE_FAMILY_METADATA_COLUMNS } from 'shared/utils/constants' const VIEW_ALL_PAGES = [{ name: 'Broad', downloadName: 'All', path: 'all' }] const COLUMNS = [ - { name: 'data_type' }, - { name: 'date_data_generation', format: ({ date_data_generation: date }) => date && new Date(date).toLocaleDateString() }, - { name: 'phenotype_description' }, - { name: 'consanguinity' }, - { - name: 'analysisStatus', - content: 'analysis_status', - format: ({ analysisStatus }) => FAMILY_ANALYSIS_STATUS_LOOKUP[analysisStatus]?.name, - }, - { name: 'solve_status' }, + ...BASE_FAMILY_METADATA_COLUMNS.map(({ secondaryExportColumn, ...col }) => col), { name: 'genes' }, { name: 'actual_inheritance' }, - { name: 'condition_id' }, - { name: 'known_condition_name' }, { name: 'individual_count', content: '# individuals' }, { name: 'family_structure' }, { name: 'proband_id' }, { name: 'paternal_id' }, { name: 'maternal_id' }, { name: 'other_individual_ids' }, - { name: 'analysis_groups' }, - { name: 'pmid_id' }, ] const FamilyMetadata = props => ( diff --git a/ui/pages/SummaryData/components/IndividualMetadata.jsx b/ui/pages/SummaryData/components/IndividualMetadata.jsx index 00bd8c31e4..6da7ac13c9 100644 --- a/ui/pages/SummaryData/components/IndividualMetadata.jsx +++ b/ui/pages/SummaryData/components/IndividualMetadata.jsx @@ -3,7 +3,7 @@ import { connect } from 'react-redux' import { getUser } from 'redux/selectors' import { BaseSemanticInput, BooleanCheckbox } from 'shared/components/form/Inputs' import LoadReportTable from 'shared/components/table/LoadReportTable' -import { FAMILY_ANALYSIS_STATUS_LOOKUP, VARIANT_METADATA_COLUMNS } from 'shared/utils/constants' +import { VARIANT_METADATA_COLUMNS, BASE_FAMILY_METADATA_COLUMNS } from 'shared/utils/constants' const ALL_PROJECTS_PATH = 'all' const GREGOR_PROJECT_PATH = 'gregor' @@ -31,29 +31,16 @@ const AIRTABLE_FIELDS = [ const CORE_COLUMNS = [ { name: 'participant_id', secondaryExportColumn: 'individual_guid' }, - { name: 'pmid_id' }, { name: 'paternal_id', secondaryExportColumn: 'paternal_guid' }, { name: 'maternal_id', secondaryExportColumn: 'maternal_guid' }, { name: 'proband_relationship' }, { name: 'sex' }, { name: 'ancestry' }, - { name: 'condition_id' }, - { name: 'known_condition_name', secondaryExportColumn: 'disorders' }, { name: 'affected_status' }, { name: 'hpo_present', style: { minWidth: '400px' } }, { name: 'hpo_absent', style: { minWidth: '400px' } }, - { name: 'phenotype_description', style: { minWidth: '200px' } }, - { name: 'analysis_groups' }, - { - name: 'analysisStatus', - content: 'analysis_status', - format: ({ analysisStatus }) => FAMILY_ANALYSIS_STATUS_LOOKUP[analysisStatus]?.name, - }, - { name: 'solve_status' }, { name: 'MME' }, - { name: 'data_type' }, - { name: 'date_data_generation', secondaryExportColumn: 'filter_flags' }, - { name: 'consanguinity' }, + ...BASE_FAMILY_METADATA_COLUMNS, { name: 'family_history' }, ] diff --git a/ui/shared/components/table/LoadReportTable.jsx b/ui/shared/components/table/LoadReportTable.jsx index 4a9ab52d41..923cfbe542 100644 --- a/ui/shared/components/table/LoadReportTable.jsx +++ b/ui/shared/components/table/LoadReportTable.jsx @@ -54,7 +54,7 @@ const ReportTable = React.memo(( striped collapsing horizontalScroll - downloadFileName={`${viewAllPages.find(({ path }) => path === projectGuid)?.downloadName || (data?.length && data[0][PROJECT_ID_FIELD].replace(/ /g, '_'))}_${new Date().toISOString().slice(0, 10)}_${fileName}`} + downloadFileName={`${viewAllPages.find(({ path }) => path === projectGuid)?.downloadName || (data?.length && (data[0][PROJECT_ID_FIELD] || '').replace(/ /g, '_'))}_${new Date().toISOString().slice(0, 10)}_${fileName}`} idField={idField} defaultSortColumn="family_id" emptyContent={projectGuid ? '0 cases found' : 'Select a project to view data'} diff --git a/ui/shared/utils/constants.js b/ui/shared/utils/constants.js index 882b788c1f..f34ed287a8 100644 --- a/ui/shared/utils/constants.js +++ b/ui/shared/utils/constants.js @@ -1876,6 +1876,23 @@ export const VARIANT_METADATA_COLUMNS = [ { name: 'ClinGen_allele_ID' }, ] +export const BASE_FAMILY_METADATA_COLUMNS = [ + { name: 'pmid_id' }, + { name: 'condition_id' }, + { name: 'known_condition_name', secondaryExportColumn: 'disorders' }, + { name: 'phenotype_description', style: { minWidth: '200px' } }, + { name: 'analysis_groups' }, + { + name: 'analysisStatus', + content: 'analysis_status', + format: ({ analysisStatus }) => FAMILY_ANALYSIS_STATUS_LOOKUP[analysisStatus]?.name, + }, + { name: 'solve_status' }, + { name: 'data_type' }, + { name: 'date_data_generation', secondaryExportColumn: 'filter_flags' }, + { name: 'consanguinity' }, +] + // RNAseq sample tissue type mapping export const TISSUE_DISPLAY = { WB: 'Whole Blood', From c9fe24a414f601656d07b28e950310c7ac73da0a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 27 Jun 2024 09:50:20 -0400 Subject: [PATCH 16/67] additional unused --- seqr/views/apis/summary_data_api_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index b5acdf21cc..2b71029865 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -112,7 +112,7 @@ 'partial_contribution_explained-1': 'HP:0000501|HP:0000365', 'partial_contribution_explained-2': '', 'condition_id': 'OMIM:616126', - 'condition_inheritance': 'Autosomal recessive', + 'condition_inheritance': 'Autosomal recessive', # unused 'known_condition_name': 'Immunodeficiency 38', 'ClinGen_allele_ID-1': 'CA1501729', 'ClinGen_allele_ID-2': None, From 77cb64c8ff8576300fc0355b1bd0e3434e93b7cd Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 27 Jun 2024 09:53:28 -0400 Subject: [PATCH 17/67] add condition_inheritance to metadata --- seqr/views/apis/report_api_tests.py | 2 +- seqr/views/apis/summary_data_api_tests.py | 2 +- ui/shared/utils/constants.js | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index 6ff66cbaf4..5b41b96c1d 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -1179,7 +1179,7 @@ def test_family_metadata(self): 'solve_status': 'Partially solved', 'actual_inheritance': 'unknown', 'condition_id': 'OMIM:616126', - 'condition_inheritance': 'Autosomal recessive', # unused + 'condition_inheritance': 'Autosomal recessive', 'known_condition_name': 'Immunodeficiency 38', 'date_data_generation': '2017-02-05', 'data_type': 'WES', diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index 2b71029865..b5acdf21cc 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -112,7 +112,7 @@ 'partial_contribution_explained-1': 'HP:0000501|HP:0000365', 'partial_contribution_explained-2': '', 'condition_id': 'OMIM:616126', - 'condition_inheritance': 'Autosomal recessive', # unused + 'condition_inheritance': 'Autosomal recessive', 'known_condition_name': 'Immunodeficiency 38', 'ClinGen_allele_ID-1': 'CA1501729', 'ClinGen_allele_ID-2': None, diff --git a/ui/shared/utils/constants.js b/ui/shared/utils/constants.js index f34ed287a8..e4f6838430 100644 --- a/ui/shared/utils/constants.js +++ b/ui/shared/utils/constants.js @@ -1879,7 +1879,8 @@ export const VARIANT_METADATA_COLUMNS = [ export const BASE_FAMILY_METADATA_COLUMNS = [ { name: 'pmid_id' }, { name: 'condition_id' }, - { name: 'known_condition_name', secondaryExportColumn: 'disorders' }, + { name: 'known_condition_name' }, + { name: 'condition_inheritance', secondaryExportColumn: 'disorders' }, { name: 'phenotype_description', style: { minWidth: '200px' } }, { name: 'analysis_groups' }, { From b2fc617a682745bbea97a1ca7b58b3e3499d74ec Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 27 Jun 2024 10:01:54 -0400 Subject: [PATCH 18/67] remove unused sample_id --- seqr/views/apis/summary_data_api.py | 27 ++++++++++++----------- seqr/views/apis/summary_data_api_tests.py | 2 -- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index 2c8663b76d..8e557c4a4a 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -289,24 +289,25 @@ def _add_row(row, family_id, row_type): parsed_row = {'{}-{}'.format(k, i + 1): v for k, v in discovery_row.items()} parsed_row['num_saved_variants'] = len(row) rows_by_subject_family_id[(participant_id, family_id)].update(parsed_row) - else: + elif row_type == SUBJECT_ROW_TYPE: row_key = (row['participant_id'], family_id) collaborator = row.pop('Collaborator', None) if collaborator: collaborator_map[row_key] = collaborator - if row_type == SUBJECT_ROW_TYPE: - race = row.pop('reported_race') - ancestry_detail = row.pop('ancestry_detail') - ethnicity = row.pop('reported_ethnicity') - row['ancestry'] = ethnicity or ancestry_detail or race - if 'features' in row: - row.update({ - 'hpo_present': [feature['id'] for feature in row.pop('features') or []], - 'hpo_absent': [feature['id'] for feature in row.pop('absent_features') or []], - }) - all_features.update(row['hpo_present']) - all_features.update(row['hpo_absent']) + race = row.pop('reported_race') + ancestry_detail = row.pop('ancestry_detail') + ethnicity = row.pop('reported_ethnicity') + row['ancestry'] = ethnicity or ancestry_detail or race + row.update({ + 'hpo_present': [feature['id'] for feature in row.pop('features') or []], + 'hpo_absent': [feature['id'] for feature in row.pop('absent_features') or []], + }) + all_features.update(row['hpo_present']) + all_features.update(row['hpo_absent']) rows_by_subject_family_id[row_key].update(row) + else: + row.pop('sample_id') + rows_by_subject_family_id[(row['participant_id'], family_id)].update(row) parse_anvil_metadata( projects, request.user, _add_row, max_loaded_date=request.GET.get('loadedBefore'), diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index b5acdf21cc..2044fc3019 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -36,7 +36,6 @@ "projectGuid": "R0003_test", "num_saved_variants": 2, "solve_status": "Partially solved", - "sample_id": "NA20889", # unused "gene_known_for_phenotype-1": "Candidate", "gene_known_for_phenotype-2": "Candidate", "variant_inheritance-1": "unknown", @@ -126,7 +125,6 @@ EXPECTED_SAMPLE_METADATA_ROW.update(EXPECTED_NO_AIRTABLE_SAMPLE_METADATA_ROW) EXPECTED_NO_GENE_SAMPLE_METADATA_ROW = { 'participant_id': 'NA21234', - 'sample_id': 'NA21234', 'familyGuid': 'F000014_14', 'family_id': '14', 'displayName': '14', From 178584da450e4266244e703dda1e9ddcdcc81fed Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 27 Jun 2024 10:16:32 -0400 Subject: [PATCH 19/67] add back internal_project_id --- seqr/views/apis/report_api.py | 5 +++-- seqr/views/apis/report_api_tests.py | 7 +++++-- ui/shared/components/table/LoadReportTable.jsx | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/seqr/views/apis/report_api.py b/seqr/views/apis/report_api.py index a00e965e1b..f13626681c 100644 --- a/seqr/views/apis/report_api.py +++ b/seqr/views/apis/report_api.py @@ -14,7 +14,7 @@ from seqr.views.utils.airtable_utils import AirtableSession from seqr.views.utils.anvil_metadata_utils import parse_anvil_metadata, anvil_export_airtable_fields, \ FAMILY_ROW_TYPE, SUBJECT_ROW_TYPE, SAMPLE_ROW_TYPE, DISCOVERY_ROW_TYPE, PARTICIPANT_TABLE, PHENOTYPE_TABLE, \ - EXPERIMENT_TABLE, EXPERIMENT_LOOKUP_TABLE, FINDINGS_TABLE, GENE_COLUMN + EXPERIMENT_TABLE, EXPERIMENT_LOOKUP_TABLE, FINDINGS_TABLE, GENE_COLUMN, FAMILY_INDIVIDUAL_FIELDS from seqr.views.utils.export_utils import export_multiple_files, write_multiple_files_to_gs from seqr.views.utils.json_utils import create_json_response from seqr.views.utils.permissions_utils import analyst_required, get_project_and_check_permissions, \ @@ -864,7 +864,7 @@ def _add_row(row, family_id, row_type): individuals_ids -= set(known_ids.values()) individual = proband or next(iter(individuals_by_id.values()), None) if individual: - f.update({k: individual[k] for k in ['phenotype_description', 'pmid_id', 'solve_status']}) + f.update({k: individual[k] for k in FAMILY_INDIVIDUAL_FIELDS}) sorted_samples = sorted(individuals_by_id.values(), key=lambda x: x.get('date_data_generation', '')) earliest_sample = next((s for s in [proband or {}] + sorted_samples if s.get('date_data_generation')), {}) @@ -926,6 +926,7 @@ def _add_row(row, family_id, row_type): families_by_id[family_id] = row elif row_type == SUBJECT_ROW_TYPE: participant_mme[row['participant_id']] = row.get('MME', {}) + families_by_id[family_id]['internal_project_id'] = row['internal_project_id'] elif row_type == DISCOVERY_ROW_TYPE: family = families_by_id[family_id] for variant in row: diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index 5b41b96c1d..84b1d6e3f1 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -492,6 +492,7 @@ } BASE_VARIANT_METADATA_ROW = { + 'internal_project_id': '1kg project nåme with uniçøde', 'ClinGen_allele_ID': None, 'MME': False, 'additional_family_members_with_variant': '', @@ -513,7 +514,6 @@ 'svType': None, 'sv_name': None, 'transcript': None, - # TODO missing internal_project_id } PARTICIPANT_TABLE = [ @@ -1173,6 +1173,7 @@ def test_family_metadata(self): test_row = next(r for r in response_json['rows'] if r['familyGuid'] == 'F000012_12') self.assertDictEqual(test_row, { 'projectGuid': 'R0003_test', + 'internal_project_id': 'Test Reprocessed Project', 'familyGuid': 'F000012_12', 'family_id': '12', 'displayName': '12', @@ -1196,7 +1197,6 @@ def test_family_metadata(self): 'analysisStatus': 'Q', 'analysis_groups': '', 'consanguinity': 'Unknown', - # TODO missing internal_project_id }) # Test all projects @@ -1212,6 +1212,7 @@ def test_family_metadata(self): test_row = next(r for r in response_json['rows'] if r['familyGuid'] == 'F000003_3') self.assertDictEqual(test_row, { 'projectGuid': 'R0001_1kg', + 'internal_project_id': '1kg project nåme with uniçøde', 'familyGuid': 'F000003_3', 'family_id': '3', 'displayName': '3', @@ -1345,6 +1346,7 @@ def test_variant_metadata(self): 'partial_contribution_explained': 'HP:0000501|HP:0000365', 'phenotype_contribution': 'Partial', 'projectGuid': 'R0003_test', + 'internal_project_id': 'Test Reprocessed Project', 'ref': 'TC', 'seqr_chosen_consequence': 'intron_variant', 'tags': ['Tier 1 - Novel gene and phenotype'], @@ -1372,6 +1374,7 @@ def test_variant_metadata(self): 'participant_id': 'NA20889', 'pos': 249045487, 'projectGuid': 'R0003_test', + 'internal_project_id': 'Test Reprocessed Project', 'ref': None, 'svType': 'DEL', 'sv_name': 'DEL:chr1:249045487-249045898', diff --git a/ui/shared/components/table/LoadReportTable.jsx b/ui/shared/components/table/LoadReportTable.jsx index 923cfbe542..4a9ab52d41 100644 --- a/ui/shared/components/table/LoadReportTable.jsx +++ b/ui/shared/components/table/LoadReportTable.jsx @@ -54,7 +54,7 @@ const ReportTable = React.memo(( striped collapsing horizontalScroll - downloadFileName={`${viewAllPages.find(({ path }) => path === projectGuid)?.downloadName || (data?.length && (data[0][PROJECT_ID_FIELD] || '').replace(/ /g, '_'))}_${new Date().toISOString().slice(0, 10)}_${fileName}`} + downloadFileName={`${viewAllPages.find(({ path }) => path === projectGuid)?.downloadName || (data?.length && data[0][PROJECT_ID_FIELD].replace(/ /g, '_'))}_${new Date().toISOString().slice(0, 10)}_${fileName}`} idField={idField} defaultSortColumn="family_id" emptyContent={projectGuid ? '0 cases found' : 'Select a project to view data'} From 27ae1c45a4bcabca0a80aa6fc4428c79b552c05b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 27 Jun 2024 10:27:00 -0400 Subject: [PATCH 20/67] use end in variant metadata reports --- seqr/views/apis/report_api_tests.py | 2 +- seqr/views/apis/summary_data_api_tests.py | 2 +- ui/shared/utils/constants.js | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index 84b1d6e3f1..6682340912 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -502,7 +502,7 @@ 'clinvar': None, 'condition_id': None, 'consanguinity': 'Unknown', # unused - 'end': None, # unused + 'end': None, 'hgvsc': '', 'hgvsp': '', 'method_of_discovery': 'SR-ES', diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index 2044fc3019..30dcfc54a0 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -51,7 +51,7 @@ "sv_name-2": "DEL:chr1:249045487-249045898", "chrom-2": "1", "pos-2": 249045487, - 'end-2': 249045898, # unused + 'end-2': 249045898, "maternal_id": "", "paternal_id": "", "maternal_guid": "", diff --git a/ui/shared/utils/constants.js b/ui/shared/utils/constants.js index e4f6838430..5ddcac1e36 100644 --- a/ui/shared/utils/constants.js +++ b/ui/shared/utils/constants.js @@ -1858,6 +1858,7 @@ export const VARIANT_METADATA_COLUMNS = [ { name: 'variant_reference_assembly' }, { name: 'chrom' }, { name: 'pos' }, + { name: 'end' }, { name: 'ref' }, { name: 'alt' }, { name: 'gene_of_interest' }, From aff110714ce5fb351edd64a2f2bf3a7c771228e1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 27 Jun 2024 10:30:56 -0400 Subject: [PATCH 21/67] remove unused svName from metadata --- seqr/views/apis/report_api_tests.py | 1 - seqr/views/apis/summary_data_api_tests.py | 5 +---- seqr/views/utils/anvil_metadata_utils.py | 3 ++- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index 6682340912..e24bd3ba1d 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -510,7 +510,6 @@ 'phenotype_contribution': 'Full', 'partial_contribution_explained': '', 'seqr_chosen_consequence': None, - 'svName': None, # unused 'svType': None, 'sv_name': None, 'transcript': None, diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index 30dcfc54a0..7f32e2443d 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -70,7 +70,7 @@ "chrom-1": "1", "alt-1": "T", "gene_of_interest-1": "OR4G11P", - "gene_id-1": "ENSG00000240361", # unused + "gene_id-1": "ENSG00000240361", 'variant_reference_assembly-1': 'GRCh37', 'variant_reference_assembly-2': 'GRCh37', "pmid_id": None, @@ -95,10 +95,8 @@ 'seqr_chosen_consequence-2': None, 'gene_of_interest-2': None, 'gene_id-2': None, - 'svName-2': None, # unused 'svType-1': None, 'sv_name-1': None, - 'svName-1': None, 'end-1': None, 'allele_balance_or_heteroplasmy_percentage-1': None, # unused 'allele_balance_or_heteroplasmy_percentage-2': None, @@ -171,7 +169,6 @@ 'hgvsp-1': '', 'notes-1': None, 'seqr_chosen_consequence-1': None, - 'svName-1': None, 'svType-1': None, 'sv_name-1': None, 'transcript-1': None, diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 4c1f58cad7..ba5f61def3 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -271,8 +271,9 @@ def _get_nested_variant_name(v): def _get_sv_name(variant_json): + sv_name = variant_json.pop('svName', None) if variant_json.get('svType'): - return variant_json.get('svName') or '{svType}:chr{chrom}:{pos}-{end}'.format(**variant_json) + return sv_name or '{svType}:chr{chrom}:{pos}-{end}'.format(**variant_json) return None From cae8a8cfc74807b56db62eb90a77e1e4b6d5e021 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 27 Jun 2024 10:33:49 -0400 Subject: [PATCH 22/67] cleanup --- seqr/views/utils/anvil_metadata_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index ba5f61def3..5011e2eb44 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -267,11 +267,11 @@ def parse_anvil_metadata( def _get_nested_variant_name(v): - return _get_sv_name(v) or f"{v['chrom']}-{v['pos']}-{v['ref']}-{v['alt']}" + return _get_sv_name(v, pop_sv_name=False) or f"{v['chrom']}-{v['pos']}-{v['ref']}-{v['alt']}" -def _get_sv_name(variant_json): - sv_name = variant_json.pop('svName', None) +def _get_sv_name(variant_json, pop_sv_name=True): + sv_name = variant_json.pop('svName', None) if pop_sv_name else variant_json.get('svName') if variant_json.get('svType'): return sv_name or '{svType}:chr{chrom}:{pos}-{end}'.format(**variant_json) return None From 3735f7988c595aee5be1b83739a1e9f05c82b33d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 27 Jun 2024 11:40:44 -0400 Subject: [PATCH 23/67] note breakdown for include_metadata --- seqr/views/apis/report_api_tests.py | 4 ++-- seqr/views/utils/anvil_metadata_utils.py | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index e24bd3ba1d..7624509b1b 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -1189,7 +1189,7 @@ def test_family_metadata(self): 'other_individual_ids': 'NA20870; NA20888', 'individual_count': 3, 'family_structure': 'other', - 'family_history': 'Yes', + 'family_history': 'Yes', # unused 'genes': 'DEL:chr1:249045487-249045898; OR4G11P', 'pmid_id': None, 'phenotype_description': None, @@ -1365,7 +1365,7 @@ def test_variant_metadata(self): 'end': 249045898, 'familyGuid': 'F000012_12', 'family_id': '12', - 'family_history': 'Yes', + 'family_history': 'Yes', # unused 'gene_of_interest': None, 'gene_id': None, 'gene_known_for_phenotype': 'Candidate', diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 5011e2eb44..0bba83abc1 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -132,14 +132,14 @@ def _get_family_metadata(family_filter, family_fields, include_metadata, include Value('\t'), Value(' '), ), analysisStatus=F('analysis_status'), - **(METADATA_FAMILY_VALUES if include_metadata else {}), + **(METADATA_FAMILY_VALUES if include_metadata else {}), # TODO analysis_groups: individual/family, rest all **{k: v['value'] for k, v in (family_fields or {}).items()} ) family_data_by_id = {} for f in family_data: family_id = f.pop('id') - analysis_status = f['analysisStatus'] if include_metadata else f.pop('analysisStatus') + analysis_status = f['analysisStatus'] if include_metadata else f.pop('analysisStatus') # TODO individual/family solve_status = ANALYSIS_SOLVE_STATUS_LOOKUP.get(analysis_status, Individual.UNSOLVED) f.update({ 'solve_status': Individual.SOLVE_STATUS_LOOKUP[solve_status], @@ -148,13 +148,12 @@ def _get_family_metadata(family_filter, family_fields, include_metadata, include if format_id: f.update({k: format_id(f[k]) for k in ['family_id', 'internal_project_id']}) if include_metadata: - f['analysis_groups'] = '; '.join(f['analysis_groups']) + f['analysis_groups'] = '; '.join(f['analysis_groups']) # TODO individual/family family_data_by_id[family_id] = f return family_data_by_id -# TODO clean up args def parse_anvil_metadata( projects: Iterable[Project], user: User, add_row: Callable[[dict, str, str], None], max_loaded_date: str = None, family_fields: dict = None, format_id: Callable[[str], str] = lambda s: s, @@ -194,7 +193,7 @@ def parse_anvil_metadata( list(sample_ids) or [i[0] for i in individual_ids_map.values()], user, airtable_fields) matchmaker_individuals = {m['individual_id']: m for m in MatchmakerSubmission.objects.filter( - individual__in=individual_samples).values('individual_id', **(mme_values or {}))} if include_metadata else {} + individual__in=individual_samples).values('individual_id', **(mme_values or {}))} if include_metadata else {} # TODO individual/variant, already dropped for family for family_id, family_subject_row in family_data_by_id.items(): saved_variants = saved_variants_by_family[family_id] @@ -207,7 +206,7 @@ def parse_anvil_metadata( affected_individuals = [ individual for individual in family_individuals if individual.affected == Individual.AFFECTED_STATUS_AFFECTED - ] if include_metadata else [] + ] if include_metadata else [] # TODO individual only subject_family_row = {k: family_subject_row.pop(k) for k in FAMILY_INDIVIDUAL_FIELDS} family_row = { @@ -377,8 +376,8 @@ def _get_parsed_saved_discovery_variants_by_family( } if include_metadata: parsed_variant.update({ - 'seqr_chosen_consequence': main_transcript.get('majorConsequence'), - 'tags': variant.tags, + 'seqr_chosen_consequence': main_transcript.get('majorConsequence'), # TODO individual/variant, currently not in family + 'tags': variant.tags, # TODO variant only }) variants.append(parsed_variant) @@ -386,7 +385,7 @@ def _get_parsed_saved_discovery_variants_by_family( saved_variants_by_family = defaultdict(list) for row in variants: - gene_id = row['gene_id'] if include_metadata else row.pop('gene_id') + gene_id = row['gene_id'] if include_metadata else row.pop('gene_id') # TODO individual only, currently not in family, add to variant? row[GENE_COLUMN] = genes_by_id.get(gene_id, {}).get('geneSymbol') family_id = row.pop('family_id') saved_variants_by_family[family_id].append(row) @@ -464,6 +463,7 @@ def _get_sample_row(sample, participant_id, has_dbgap_submission, airtable_metad if has_dbgap_submission: sample_row['dbgap_sample_id'] = airtable_metadata.get('dbgap_sample_id', '') if include_metadata: + # TODO individual/family, currently not in variant sample_row.update({ 'data_type': sample.sample_type, 'date_data_generation': sample.loaded_date.strftime('%Y-%m-%d'), From eb4cb9a011cc27ffc08e85c524af6b565d29b4fe Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 27 Jun 2024 11:45:59 -0400 Subject: [PATCH 24/67] only return tags in variant metadta --- seqr/views/apis/report_api.py | 1 + seqr/views/apis/summary_data_api_tests.py | 3 --- seqr/views/utils/anvil_metadata_utils.py | 8 ++++---- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/seqr/views/apis/report_api.py b/seqr/views/apis/report_api.py index f13626681c..1557d09628 100644 --- a/seqr/views/apis/report_api.py +++ b/seqr/views/apis/report_api.py @@ -944,6 +944,7 @@ def _add_row(row, family_id, row_type): individual_data_types={i.individual_id: i.data_types for i in individuals}, add_row=_add_row, variant_json_fields=['clinvar', 'variantId'], + variant_attr_fields=['tags'], mme_values={'variant_ids': ArrayAgg('matchmakersubmissiongenes__saved_variant__saved_variant_json__variantId')}, include_metadata=True, include_mondo=True, diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index 7f32e2443d..eac67d64f8 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -102,8 +102,6 @@ 'allele_balance_or_heteroplasmy_percentage-2': None, 'notes-1': None, 'notes-2': None, - 'tags-1': ['Tier 1 - Novel gene and phenotype'], # unused - 'tags-2': ['Tier 1 - Novel gene and phenotype'], 'phenotype_contribution-1': 'Partial', 'phenotype_contribution-2': 'Full', 'partial_contribution_explained-1': 'HP:0000501|HP:0000365', @@ -154,7 +152,6 @@ 'alt-1': 'T', 'chrom-1': '1', 'gene_known_for_phenotype-1': 'Candidate', - 'tags-1': ['Tier 1 - Novel gene and phenotype'], 'phenotype_contribution-1': 'Full', 'partial_contribution_explained-1': '', 'pos-1': 248367227, diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 0bba83abc1..6178157b89 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -161,7 +161,7 @@ def parse_anvil_metadata( get_additional_individual_fields: Callable[[Individual, dict], dict] = None, individual_samples: dict[Individual, Sample] = None, individual_data_types: dict[str, Iterable[str]] = None, airtable_fields: Iterable[str] = None, mme_values: dict = None, include_svs: bool = True, - variant_json_fields: Iterable[str] = None, post_process_variant: Callable[[dict, list[dict]], dict] = None, + variant_json_fields: Iterable[str] = None, variant_attr_fields: Iterable[str] = None, post_process_variant: Callable[[dict, list[dict]], dict] = None, include_no_individual_families: bool = False, omit_airtable: bool = False, include_metadata: bool = False, include_discovery_sample_id: bool = False, include_mondo: bool = False, include_parent_mnvs: bool = False, proband_only_variants: bool = False): @@ -184,7 +184,7 @@ def parse_anvil_metadata( sample_ids.add(sample.sample_id) saved_variants_by_family = _get_parsed_saved_discovery_variants_by_family( - list(family_data_by_id.keys()), include_metadata, include_svs=include_svs, variant_json_fields=variant_json_fields, + list(family_data_by_id.keys()), include_metadata, include_svs, variant_json_fields, variant_attr_fields, ) condition_map = _get_condition_map(family_data_by_id.values()) @@ -329,6 +329,7 @@ def _post_process_variant_metadata(v, gene_variants, include_parent_mnvs=False): def _get_parsed_saved_discovery_variants_by_family( families: Iterable[Family], include_metadata: bool, include_svs: dict, variant_json_fields: list[str], + variant_attr_fields: list[str], ): tag_types = VariantTagType.objects.filter(project__isnull=True, category=DISCOVERY_CATEGORY) @@ -372,12 +373,11 @@ def _get_parsed_saved_discovery_variants_by_family( **{k: _get_transcript_field(k, config, main_transcript) for k, config in TRANSCRIPT_FIELDS.items()}, **{k: variant_json.get(k) for k in variant_fields + (variant_json_fields or [])}, 'ClinGen_allele_ID': variant_json.get('CAID'), - **{k: getattr(variant, k) for k in ['family_id', 'ref', 'alt']}, + **{k: getattr(variant, k) for k in ['family_id', 'ref', 'alt'] + (variant_attr_fields or [])}, } if include_metadata: parsed_variant.update({ 'seqr_chosen_consequence': main_transcript.get('majorConsequence'), # TODO individual/variant, currently not in family - 'tags': variant.tags, # TODO variant only }) variants.append(parsed_variant) From 76ccd224139c069c667171d82b110c2493c257c6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 27 Jun 2024 11:57:06 -0400 Subject: [PATCH 25/67] clean up family_history computation --- seqr/views/apis/report_api_tests.py | 3 --- seqr/views/apis/summary_data_api.py | 4 ++++ seqr/views/utils/anvil_metadata_utils.py | 6 ------ 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index 7624509b1b..8f65cf7dbe 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -1189,7 +1189,6 @@ def test_family_metadata(self): 'other_individual_ids': 'NA20870; NA20888', 'individual_count': 3, 'family_structure': 'other', - 'family_history': 'Yes', # unused 'genes': 'DEL:chr1:249045487-249045898; OR4G11P', 'pmid_id': None, 'phenotype_description': None, @@ -1333,7 +1332,6 @@ def test_variant_metadata(self): 'displayName': '12', 'familyGuid': 'F000012_12', 'family_id': '12', - 'family_history': 'Yes', # unused 'gene_of_interest': 'OR4G11P', 'gene_id': 'ENSG00000240361', 'gene_known_for_phenotype': 'Candidate', @@ -1365,7 +1363,6 @@ def test_variant_metadata(self): 'end': 249045898, 'familyGuid': 'F000012_12', 'family_id': '12', - 'family_history': 'Yes', # unused 'gene_of_interest': None, 'gene_id': None, 'gene_known_for_phenotype': 'Candidate', diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index 8e557c4a4a..7c4bdb2f7e 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -294,6 +294,9 @@ def _add_row(row, family_id, row_type): collaborator = row.pop('Collaborator', None) if collaborator: collaborator_map[row_key] = collaborator + is_additional_affected = row.pop('is_additional_affected') + if is_additional_affected: + family_rows_by_id[family_id]['family_history'] = 'Yes' race = row.pop('reported_race') ancestry_detail = row.pop('ancestry_detail') ethnicity = row.pop('reported_ethnicity') @@ -320,6 +323,7 @@ def _add_row(row, family_id, row_type): 'filter_flags': json.dumps(individual.filter_flags) if individual.filter_flags else '', 'paternal_guid': paternal_ids[1], 'maternal_guid': maternal_ids[1], + 'is_additional_affected': individual.affected == Individual.AFFECTED_STATUS_AFFECTED and individual.proband_relationship != Individual.SELF_RELATIONSHIP, **anvil_export_airtable_fields(airtable_metadata, has_dbgap_submission), }, ) diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 6178157b89..d0e003b47b 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -204,10 +204,6 @@ def parse_anvil_metadata( family_subject_row, saved_variants, *condition_map, set_conditions_for_variants=proband_only_variants, ) - affected_individuals = [ - individual for individual in family_individuals if individual.affected == Individual.AFFECTED_STATUS_AFFECTED - ] if include_metadata else [] # TODO individual only - subject_family_row = {k: family_subject_row.pop(k) for k in FAMILY_INDIVIDUAL_FIELDS} family_row = { 'family_id': subject_family_row['family_id'], @@ -217,8 +213,6 @@ def parse_anvil_metadata( ), 'Unknown'), **family_subject_row, } - if len(affected_individuals) > 1: - family_row['family_history'] = 'Yes' add_row(family_row, family_id, FAMILY_ROW_TYPE) for individual in family_individuals: From 4f810b0847c9a2635aa15e01b35266f58f56eb69 Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Thu, 27 Jun 2024 12:01:26 -0400 Subject: [PATCH 26/67] hopefully just these changes --- requirements.in | 2 +- requirements.txt | 6 ++---- seqr/utils/middleware.py | 2 +- settings.py | 9 ++++++--- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/requirements.in b/requirements.in index 9ddc62799f..4a76de1a1f 100644 --- a/requirements.in +++ b/requirements.in @@ -1,4 +1,4 @@ -Django<3.3 # core server-side framework +Django==4.2 # core server-side framework django-anymail # for sending emails using cloud-based mail service providers django-csp # for setting CSP headers django-guardian # object-level permissions for database records. Behind a major version due to missing Python 2 support diff --git a/requirements.txt b/requirements.txt index 8e7ef47708..83322e1a7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,7 @@ defusedxml==0.7.1 # via # python3-openid # social-auth-core -django==3.2.25 +django==4.2 # via # -r requirements.in # django-anymail @@ -126,9 +126,7 @@ python-dateutil==2.8.2 python3-openid==3.2.0 # via social-auth-core pytz==2022.7.1 - # via - # django - # django-notifications-hq + # via django-notifications-hq redis==4.5.4 # via -r requirements.in requests==2.32.2 diff --git a/seqr/utils/middleware.py b/seqr/utils/middleware.py index 1ee3e53195..33d22532a3 100644 --- a/seqr/utils/middleware.py +++ b/seqr/utils/middleware.py @@ -104,7 +104,7 @@ def process_response(request, response): # conforms to the httpRequest json spec for stackdriver: https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#HttpRequest http_json = { 'requestMethod': request.method, - 'requestUrl': request.get_raw_uri(), + 'requestUrl': request.build_absolute_uri(), 'status': response.status_code, 'responseSize': len(response.content) if hasattr(response, 'content') else request.META.get('CONTENT_LENGTH'), 'userAgent': request.META.get('HTTP_USER_AGENT'), diff --git a/settings.py b/settings.py index b7fd626bfa..430c9082a1 100644 --- a/settings.py +++ b/settings.py @@ -128,10 +128,12 @@ USE_TZ = True # Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/1.10/howto/static-files/ +# https://docs.djangoproject.com/en/4.2/howto/static-files/ STATIC_URL = '/static/' -STATICFILES_DIRS = ['ui/dist'] -STATIC_ROOT = os.path.join(BASE_DIR, 'static') +STATICFILES_DIRS = [ + os.path.join(BASE_DIR, 'static'), + 'ui/dist', +] STATICFILES_FINDERS = ( 'django.contrib.staticfiles.finders.FileSystemFinder', 'django.contrib.staticfiles.finders.AppDirectoriesFinder', @@ -262,6 +264,7 @@ 'http://localhost:3000', 'http://localhost:8000', ) + # TODO: will docker build fail if STATICFILES_DIRS always contains both? # the collectstatic step in docker build runs without env variables set, and uncommenting these lines breaks the docker build # STATICFILES_DIRS.append(STATIC_ROOT) # STATIC_ROOT = None From 4be34fd04f40a35cffc1abdef3416daa0f4870a1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 27 Jun 2024 12:10:36 -0400 Subject: [PATCH 27/67] clean up mme annotation --- seqr/views/apis/report_api.py | 4 ++-- seqr/views/apis/summary_data_api.py | 1 + seqr/views/utils/anvil_metadata_utils.py | 10 +++++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/seqr/views/apis/report_api.py b/seqr/views/apis/report_api.py index 1557d09628..069032c125 100644 --- a/seqr/views/apis/report_api.py +++ b/seqr/views/apis/report_api.py @@ -931,7 +931,7 @@ def _add_row(row, family_id, row_type): family = families_by_id[family_id] for variant in row: variant_rows.append({ - 'MME': variant.pop('variantId') in participant_mme[variant['participant_id']].get('variant_ids', []), + 'MME': variant.pop('variantId') in (participant_mme[variant['participant_id']] or []), 'phenotype_contribution': 'Full', **family, **variant, @@ -945,7 +945,7 @@ def _add_row(row, family_id, row_type): add_row=_add_row, variant_json_fields=['clinvar', 'variantId'], variant_attr_fields=['tags'], - mme_values={'variant_ids': ArrayAgg('matchmakersubmissiongenes__saved_variant__saved_variant_json__variantId')}, + mme_value=ArrayAgg('matchmakersubmissiongenes__saved_variant__saved_variant_json__variantId'), include_metadata=True, include_mondo=True, omit_airtable=True, diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index 7c4bdb2f7e..022480ce0c 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -316,6 +316,7 @@ def _add_row(row, family_id, row_type): projects, request.user, _add_row, max_loaded_date=request.GET.get('loadedBefore'), include_metadata=True, omit_airtable=not include_airtable, + mme_value=Value('Yes'), get_additional_individual_fields=lambda individual, airtable_metadata, has_dbgap_submission, maternal_ids, paternal_ids: { 'Collaborator': (airtable_metadata or {}).get('Collaborator'), 'individual_guid': individual.guid, diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index d0e003b47b..ee87e168db 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -1,6 +1,6 @@ from collections import defaultdict from datetime import datetime -from django.db.models import F, Q, Value, CharField +from django.db.models import F, Q, Value, CharField, Aggregate from django.db.models.functions import Replace from django.contrib.auth.models import User from django.contrib.postgres.aggregates import ArrayAgg @@ -160,7 +160,7 @@ def parse_anvil_metadata( get_additional_sample_fields: Callable[[Sample, dict], dict] = None, get_additional_individual_fields: Callable[[Individual, dict], dict] = None, individual_samples: dict[Individual, Sample] = None, individual_data_types: dict[str, Iterable[str]] = None, - airtable_fields: Iterable[str] = None, mme_values: dict = None, include_svs: bool = True, + airtable_fields: Iterable[str] = None, mme_value: Aggregate = None, include_svs: bool = True, variant_json_fields: Iterable[str] = None, variant_attr_fields: Iterable[str] = None, post_process_variant: Callable[[dict, list[dict]], dict] = None, include_no_individual_families: bool = False, omit_airtable: bool = False, include_metadata: bool = False, include_discovery_sample_id: bool = False, include_mondo: bool = False, include_parent_mnvs: bool = False, @@ -192,8 +192,8 @@ def parse_anvil_metadata( sample_airtable_metadata = None if omit_airtable else _get_sample_airtable_metadata( list(sample_ids) or [i[0] for i in individual_ids_map.values()], user, airtable_fields) - matchmaker_individuals = {m['individual_id']: m for m in MatchmakerSubmission.objects.filter( - individual__in=individual_samples).values('individual_id', **(mme_values or {}))} if include_metadata else {} # TODO individual/variant, already dropped for family + matchmaker_individuals = {m['individual_id']: m['value'] for m in MatchmakerSubmission.objects.filter( + individual__in=individual_samples).values('individual_id', value=mme_value)} if mme_value else {} for family_id, family_subject_row in family_data_by_id.items(): saved_variants = saved_variants_by_family[family_id] @@ -233,7 +233,7 @@ def parse_anvil_metadata( format_id, ) if individual.id in matchmaker_individuals: - subject_row['MME'] = matchmaker_individuals[individual.id] if mme_values else 'Yes' + subject_row['MME'] = matchmaker_individuals[individual.id] subject_row.update(subject_family_row) if individual.solve_status: subject_row['solve_status'] = Individual.SOLVE_STATUS_LOOKUP[individual.solve_status] From f006ded12856725998fe74a1226fcd7863c7c69b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 27 Jun 2024 12:25:02 -0400 Subject: [PATCH 28/67] specific family metadata fields --- seqr/views/apis/report_api.py | 4 +++- seqr/views/apis/report_api_tests.py | 2 -- seqr/views/apis/summary_data_api.py | 2 ++ seqr/views/utils/anvil_metadata_utils.py | 26 ++++++++++++------------ 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/seqr/views/apis/report_api.py b/seqr/views/apis/report_api.py index 069032c125..70d02cca9f 100644 --- a/seqr/views/apis/report_api.py +++ b/seqr/views/apis/report_api.py @@ -847,7 +847,8 @@ def _add_row(row, family_id, row_type): family['inheritance_models'].update({v['variant_inheritance'] for v in row}) parse_anvil_metadata( - projects, user=request.user, add_row=_add_row, omit_airtable=True, include_metadata=True, include_no_individual_families=True) + projects, user=request.user, add_row=_add_row, omit_airtable=True, include_metadata=True, + include_family_name_display=True, include_family_sample_metadata=True, include_no_individual_families=True) for family_id, f in families_by_id.items(): individuals_by_id = family_individuals[family_id] @@ -947,6 +948,7 @@ def _add_row(row, family_id, row_type): variant_attr_fields=['tags'], mme_value=ArrayAgg('matchmakersubmissiongenes__saved_variant__saved_variant_json__variantId'), include_metadata=True, + include_family_name_display=True, include_mondo=True, omit_airtable=True, proband_only_variants=True, diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index 8f65cf7dbe..5b74a65121 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -497,8 +497,6 @@ 'MME': False, 'additional_family_members_with_variant': '', 'allele_balance_or_heteroplasmy_percentage': None, - 'analysisStatus': 'Q', # unused - 'analysis_groups': '', # unused 'clinvar': None, 'condition_id': None, 'consanguinity': 'Unknown', # unused diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index 022480ce0c..c2fc6ff734 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -315,6 +315,8 @@ def _add_row(row, family_id, row_type): parse_anvil_metadata( projects, request.user, _add_row, max_loaded_date=request.GET.get('loadedBefore'), include_metadata=True, + include_family_name_display=True, + include_family_sample_metadata=True, omit_airtable=not include_airtable, mme_value=Value('Yes'), get_additional_individual_fields=lambda individual, airtable_metadata, has_dbgap_submission, maternal_ids, paternal_ids: { diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index ee87e168db..2e3851bd76 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -89,11 +89,10 @@ SAMPLE_ROW_TYPE = 'sample' DISCOVERY_ROW_TYPE = 'discovery' -METADATA_FAMILY_VALUES = { +FAMILY_NAME_DISPLAY_VALUES = { 'familyGuid': F('guid'), 'projectGuid': F('project__guid'), 'displayName': F('family_id'), - 'analysis_groups': ArrayAgg('analysisgroup__name', distinct=True, filter=Q(analysisgroup__isnull=False)), } METHOD_MAP = { @@ -121,7 +120,11 @@ def _format_transcript_id(transcript_id, transcript): } -def _get_family_metadata(family_filter, family_fields, include_metadata, include_mondo, format_id): +def _get_family_metadata(family_filter, family_fields, include_family_name_display, include_family_sample_metadata, include_mondo, format_id): + family_fields = {'analysis_groups': { + 'value': ArrayAgg('analysisgroup__name', distinct=True, filter=Q(analysisgroup__isnull=False)), + 'format': lambda f: '; '.join(f['analysis_groups']), + }} if include_family_sample_metadata else family_fields family_data = Family.objects.filter(**family_filter).distinct().order_by('id').values( 'id', 'family_id', 'post_discovery_omim_numbers', *(['post_discovery_mondo_id'] if include_mondo else []), @@ -132,14 +135,14 @@ def _get_family_metadata(family_filter, family_fields, include_metadata, include Value('\t'), Value(' '), ), analysisStatus=F('analysis_status'), - **(METADATA_FAMILY_VALUES if include_metadata else {}), # TODO analysis_groups: individual/family, rest all + **(FAMILY_NAME_DISPLAY_VALUES if include_family_name_display else {}), **{k: v['value'] for k, v in (family_fields or {}).items()} ) family_data_by_id = {} for f in family_data: family_id = f.pop('id') - analysis_status = f['analysisStatus'] if include_metadata else f.pop('analysisStatus') # TODO individual/family + analysis_status = f['analysisStatus'] if include_family_sample_metadata else f.pop('analysisStatus') solve_status = ANALYSIS_SOLVE_STATUS_LOOKUP.get(analysis_status, Individual.UNSOLVED) f.update({ 'solve_status': Individual.SOLVE_STATUS_LOOKUP[solve_status], @@ -147,8 +150,6 @@ def _get_family_metadata(family_filter, family_fields, include_metadata, include }) if format_id: f.update({k: format_id(f[k]) for k in ['family_id', 'internal_project_id']}) - if include_metadata: - f['analysis_groups'] = '; '.join(f['analysis_groups']) # TODO individual/family family_data_by_id[family_id] = f return family_data_by_id @@ -162,7 +163,7 @@ def parse_anvil_metadata( individual_samples: dict[Individual, Sample] = None, individual_data_types: dict[str, Iterable[str]] = None, airtable_fields: Iterable[str] = None, mme_value: Aggregate = None, include_svs: bool = True, variant_json_fields: Iterable[str] = None, variant_attr_fields: Iterable[str] = None, post_process_variant: Callable[[dict, list[dict]], dict] = None, - include_no_individual_families: bool = False, omit_airtable: bool = False, include_metadata: bool = False, + include_no_individual_families: bool = False, omit_airtable: bool = False, include_metadata: bool = False, include_family_name_display: bool = False, include_family_sample_metadata: bool = False, include_discovery_sample_id: bool = False, include_mondo: bool = False, include_parent_mnvs: bool = False, proband_only_variants: bool = False): @@ -171,7 +172,7 @@ def parse_anvil_metadata( family_data_by_id = _get_family_metadata( {'project__in': projects} if include_no_individual_families else {'individual__in': individual_samples}, - family_fields, include_metadata, include_mondo, format_id + family_fields, include_family_name_display, include_family_sample_metadata, include_mondo, format_id ) individuals_by_family_id = defaultdict(list) @@ -243,7 +244,7 @@ def parse_anvil_metadata( participant_id = subject_row['participant_id'] if sample: - sample_row = _get_sample_row(sample, participant_id, has_dbgap_submission, airtable_metadata, include_metadata, get_additional_sample_fields) + sample_row = _get_sample_row(sample, participant_id, has_dbgap_submission, airtable_metadata, include_family_sample_metadata, get_additional_sample_fields) add_row(sample_row, family_id, SAMPLE_ROW_TYPE) if proband_only_variants and individual.proband_relationship != Individual.SELF_RELATIONSHIP: @@ -449,15 +450,14 @@ def anvil_export_airtable_fields(airtable_metadata, has_dbgap_submission): } -def _get_sample_row(sample, participant_id, has_dbgap_submission, airtable_metadata, include_metadata, get_additional_sample_fields=None): +def _get_sample_row(sample, participant_id, has_dbgap_submission, airtable_metadata, include_family_sample_metadata, get_additional_sample_fields=None): sample_row = { 'participant_id': participant_id, 'sample_id': sample.sample_id, } if has_dbgap_submission: sample_row['dbgap_sample_id'] = airtable_metadata.get('dbgap_sample_id', '') - if include_metadata: - # TODO individual/family, currently not in variant + if include_family_sample_metadata: sample_row.update({ 'data_type': sample.sample_type, 'date_data_generation': sample.loaded_date.strftime('%Y-%m-%d'), From 82d10f05476e514303c9f10190fdc6dedaf7498a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 27 Jun 2024 12:29:19 -0400 Subject: [PATCH 29/67] clean up --- seqr/views/apis/report_api.py | 2 +- seqr/views/apis/summary_data_api.py | 1 - seqr/views/utils/anvil_metadata_utils.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/seqr/views/apis/report_api.py b/seqr/views/apis/report_api.py index 70d02cca9f..87fb635e19 100644 --- a/seqr/views/apis/report_api.py +++ b/seqr/views/apis/report_api.py @@ -848,7 +848,7 @@ def _add_row(row, family_id, row_type): parse_anvil_metadata( projects, user=request.user, add_row=_add_row, omit_airtable=True, include_metadata=True, - include_family_name_display=True, include_family_sample_metadata=True, include_no_individual_families=True) + include_family_sample_metadata=True, include_no_individual_families=True) for family_id, f in families_by_id.items(): individuals_by_id = family_individuals[family_id] diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index c2fc6ff734..c04136c4fb 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -315,7 +315,6 @@ def _add_row(row, family_id, row_type): parse_anvil_metadata( projects, request.user, _add_row, max_loaded_date=request.GET.get('loadedBefore'), include_metadata=True, - include_family_name_display=True, include_family_sample_metadata=True, omit_airtable=not include_airtable, mme_value=Value('Yes'), diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 2e3851bd76..c2243a7a55 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -135,7 +135,7 @@ def _get_family_metadata(family_filter, family_fields, include_family_name_displ Value('\t'), Value(' '), ), analysisStatus=F('analysis_status'), - **(FAMILY_NAME_DISPLAY_VALUES if include_family_name_display else {}), + **(FAMILY_NAME_DISPLAY_VALUES if include_family_name_display or include_family_sample_metadata else {}), **{k: v['value'] for k, v in (family_fields or {}).items()} ) From bb847253daa0a16de27cd7862dfcb37465652dca Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Thu, 27 Jun 2024 12:30:59 -0400 Subject: [PATCH 30/67] conditionally set static root --- settings.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/settings.py b/settings.py index 430c9082a1..5598d9dadd 100644 --- a/settings.py +++ b/settings.py @@ -127,18 +127,6 @@ USE_L10N = True USE_TZ = True -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/4.2/howto/static-files/ -STATIC_URL = '/static/' -STATICFILES_DIRS = [ - os.path.join(BASE_DIR, 'static'), - 'ui/dist', -] -STATICFILES_FINDERS = ( - 'django.contrib.staticfiles.finders.FileSystemFinder', - 'django.contrib.staticfiles.finders.AppDirectoriesFinder', -) - # If specified, store data in the named GCS bucket and use the gcloud storage backend. # Else, fall back to a path on the local filesystem. GCS_MEDIA_ROOT_BUCKET = os.environ.get('GCS_MEDIA_ROOT_BUCKET') @@ -264,7 +252,7 @@ 'http://localhost:3000', 'http://localhost:8000', ) - # TODO: will docker build fail if STATICFILES_DIRS always contains both? + # TODO: ? # the collectstatic step in docker build runs without env variables set, and uncommenting these lines breaks the docker build # STATICFILES_DIRS.append(STATIC_ROOT) # STATIC_ROOT = None @@ -276,6 +264,22 @@ HIJACK_LOGIN_REDIRECT_URL = '/' TEMPLATE_DIRS.append('ui') + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/4.2/howto/static-files/ +STATIC_URL = '/static/' +STATICFILES_DIRS = ['ui/dist'] +if DEBUG: + STATICFILES_DIRS.append(os.path.join(BASE_DIR, 'static')) +else: + STATIC_ROOT = os.path.join(BASE_DIR, 'static') + +STATICFILES_FINDERS = ( + 'django.contrib.staticfiles.finders.FileSystemFinder', + 'django.contrib.staticfiles.finders.AppDirectoriesFinder', +) + + TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', From 30ee8df2972b3cd4a45609365fa7de9ef7c51b9a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 27 Jun 2024 12:32:04 -0400 Subject: [PATCH 31/67] final include_metadata cleanup --- seqr/views/apis/report_api.py | 4 +--- seqr/views/apis/summary_data_api.py | 1 - seqr/views/utils/anvil_metadata_utils.py | 8 ++++---- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/seqr/views/apis/report_api.py b/seqr/views/apis/report_api.py index 87fb635e19..6a6948a0fb 100644 --- a/seqr/views/apis/report_api.py +++ b/seqr/views/apis/report_api.py @@ -847,8 +847,7 @@ def _add_row(row, family_id, row_type): family['inheritance_models'].update({v['variant_inheritance'] for v in row}) parse_anvil_metadata( - projects, user=request.user, add_row=_add_row, omit_airtable=True, include_metadata=True, - include_family_sample_metadata=True, include_no_individual_families=True) + projects, user=request.user, add_row=_add_row, omit_airtable=True, include_family_sample_metadata=True, include_no_individual_families=True) for family_id, f in families_by_id.items(): individuals_by_id = family_individuals[family_id] @@ -947,7 +946,6 @@ def _add_row(row, family_id, row_type): variant_json_fields=['clinvar', 'variantId'], variant_attr_fields=['tags'], mme_value=ArrayAgg('matchmakersubmissiongenes__saved_variant__saved_variant_json__variantId'), - include_metadata=True, include_family_name_display=True, include_mondo=True, omit_airtable=True, diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index c04136c4fb..1d988ee506 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -314,7 +314,6 @@ def _add_row(row, family_id, row_type): parse_anvil_metadata( projects, request.user, _add_row, max_loaded_date=request.GET.get('loadedBefore'), - include_metadata=True, include_family_sample_metadata=True, omit_airtable=not include_airtable, mme_value=Value('Yes'), diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index c2243a7a55..2187c8e196 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -163,7 +163,7 @@ def parse_anvil_metadata( individual_samples: dict[Individual, Sample] = None, individual_data_types: dict[str, Iterable[str]] = None, airtable_fields: Iterable[str] = None, mme_value: Aggregate = None, include_svs: bool = True, variant_json_fields: Iterable[str] = None, variant_attr_fields: Iterable[str] = None, post_process_variant: Callable[[dict, list[dict]], dict] = None, - include_no_individual_families: bool = False, omit_airtable: bool = False, include_metadata: bool = False, include_family_name_display: bool = False, include_family_sample_metadata: bool = False, + include_no_individual_families: bool = False, omit_airtable: bool = False, include_family_name_display: bool = False, include_family_sample_metadata: bool = False, include_discovery_sample_id: bool = False, include_mondo: bool = False, include_parent_mnvs: bool = False, proband_only_variants: bool = False): @@ -185,7 +185,7 @@ def parse_anvil_metadata( sample_ids.add(sample.sample_id) saved_variants_by_family = _get_parsed_saved_discovery_variants_by_family( - list(family_data_by_id.keys()), include_metadata, include_svs, variant_json_fields, variant_attr_fields, + list(family_data_by_id.keys()), bool(mme_value), include_svs, variant_json_fields, variant_attr_fields, ) condition_map = _get_condition_map(family_data_by_id.values()) @@ -372,7 +372,7 @@ def _get_parsed_saved_discovery_variants_by_family( } if include_metadata: parsed_variant.update({ - 'seqr_chosen_consequence': main_transcript.get('majorConsequence'), # TODO individual/variant, currently not in family + 'seqr_chosen_consequence': main_transcript.get('majorConsequence'), }) variants.append(parsed_variant) @@ -380,7 +380,7 @@ def _get_parsed_saved_discovery_variants_by_family( saved_variants_by_family = defaultdict(list) for row in variants: - gene_id = row['gene_id'] if include_metadata else row.pop('gene_id') # TODO individual only, currently not in family, add to variant? + gene_id = row['gene_id'] if include_metadata else row.pop('gene_id') row[GENE_COLUMN] = genes_by_id.get(gene_id, {}).get('geneSymbol') family_id = row.pop('family_id') saved_variants_by_family[family_id].append(row) From 055bf49c070a697abbc3f4abde0e24c382857ac3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 27 Jun 2024 12:33:47 -0400 Subject: [PATCH 32/67] remove unused field from inidivudal metadata --- seqr/views/apis/summary_data_api.py | 2 +- seqr/views/apis/summary_data_api_tests.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index 1d988ee506..df71a4837f 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -286,7 +286,7 @@ def _add_row(row, family_id, row_type): elif row_type == DISCOVERY_ROW_TYPE: for i, discovery_row in enumerate(row): participant_id = discovery_row.pop('participant_id') - parsed_row = {'{}-{}'.format(k, i + 1): v for k, v in discovery_row.items()} + parsed_row = {'{}-{}'.format(k, i + 1): v for k, v in discovery_row.items() if k != 'allele_balance_or_heteroplasmy_percentage'} parsed_row['num_saved_variants'] = len(row) rows_by_subject_family_id[(participant_id, family_id)].update(parsed_row) elif row_type == SUBJECT_ROW_TYPE: diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index eac67d64f8..835c9ac946 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -98,8 +98,6 @@ 'svType-1': None, 'sv_name-1': None, 'end-1': None, - 'allele_balance_or_heteroplasmy_percentage-1': None, # unused - 'allele_balance_or_heteroplasmy_percentage-2': None, 'notes-1': None, 'notes-2': None, 'phenotype_contribution-1': 'Partial', @@ -159,7 +157,6 @@ 'ref-1': 'TC', 'zygosity-1': 'Heterozygous', 'variant_reference_assembly-1': 'GRCh38', - 'allele_balance_or_heteroplasmy_percentage-1': None, 'gene_of_interest-1': None, 'gene_id-1': None, 'hgvsc-1': '', From 2eebf81e9ca086d7ac9ddb19a479660489649f5f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 27 Jun 2024 12:38:06 -0400 Subject: [PATCH 33/67] remove unused consanguinity --- seqr/views/apis/report_api_tests.py | 1 - seqr/views/utils/anvil_metadata_utils.py | 9 +++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index 5b74a65121..0bc533fb46 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -499,7 +499,6 @@ 'allele_balance_or_heteroplasmy_percentage': None, 'clinvar': None, 'condition_id': None, - 'consanguinity': 'Unknown', # unused 'end': None, 'hgvsc': '', 'hgvsp': '', diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 2187c8e196..40aca9ca8a 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -208,12 +208,13 @@ def parse_anvil_metadata( subject_family_row = {k: family_subject_row.pop(k) for k in FAMILY_INDIVIDUAL_FIELDS} family_row = { 'family_id': subject_family_row['family_id'], - 'consanguinity': next(( - 'Present' if individual.consanguinity else 'None suspected' - for individual in family_individuals if individual.consanguinity is not None - ), 'Unknown'), **family_subject_row, } + if not include_family_name_display: + family_row['consanguinity'] = next(( + 'Present' if individual.consanguinity else 'None suspected' + for individual in family_individuals if individual.consanguinity is not None + ), 'Unknown') add_row(family_row, family_id, FAMILY_ROW_TYPE) for individual in family_individuals: From a1cc0013b7df579a4b28d7c8eaf57d0dfeb5d246 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 27 Jun 2024 12:46:19 -0400 Subject: [PATCH 34/67] include gene_id in variant download --- seqr/views/apis/report_api_tests.py | 2 +- ui/pages/SummaryData/components/IndividualMetadata.jsx | 4 ++-- ui/shared/utils/constants.js | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index 0bc533fb46..d532ba142f 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -1264,7 +1264,7 @@ def test_variant_metadata(self): 'familyGuid': 'F000002_2', 'family_id': '2', 'gene_of_interest': 'RP11', - 'gene_id': 'ENSG00000135953', # unused + 'gene_id': 'ENSG00000135953', 'gene_known_for_phenotype': 'Known', 'genetic_findings_id': 'HG00731_1_248367227', 'known_condition_name': 'mitochondrial disease', diff --git a/ui/pages/SummaryData/components/IndividualMetadata.jsx b/ui/pages/SummaryData/components/IndividualMetadata.jsx index 6da7ac13c9..91c7610390 100644 --- a/ui/pages/SummaryData/components/IndividualMetadata.jsx +++ b/ui/pages/SummaryData/components/IndividualMetadata.jsx @@ -65,9 +65,9 @@ const getColumns = (data) => { const hasAirtable = data && data[0] && data[0][AIRTABLE_DBGAP_SUBMISSION_FIELD] return [...CORE_COLUMNS, ...(hasAirtable ? AIRTABLE_COLUMNS : [])].concat( ...[...Array(maxSavedVariants).keys()].map(i => VARIANT_METADATA_COLUMNS.map( - ({ name, format, fieldName, ...col }) => ({ + ({ name, format, fieldName, secondaryExportColumn, ...col }) => ({ name: `${name}-${i + 1}`, - secondaryExportColumn: name === 'gene_of_interest' ? `gene_id-${i + 1}` : null, + secondaryExportColumn: secondaryExportColumn && `${secondaryExportColumn}-${i + 1}`, format: format ? row => format({ [fieldName]: row[`${fieldName}-${i + 1}`] }) : null, ...col, }), diff --git a/ui/shared/utils/constants.js b/ui/shared/utils/constants.js index 5ddcac1e36..6f7c3faf5c 100644 --- a/ui/shared/utils/constants.js +++ b/ui/shared/utils/constants.js @@ -1861,7 +1861,7 @@ export const VARIANT_METADATA_COLUMNS = [ { name: 'end' }, { name: 'ref' }, { name: 'alt' }, - { name: 'gene_of_interest' }, + { name: 'gene_of_interest', secondaryExportColumn: 'gene_id' }, { name: 'seqr_chosen_consequence' }, { name: 'transcript' }, { name: 'hgvsc' }, From f11c73b7eaff9d92d25d2b97d5596c04147809ab Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 27 Jun 2024 12:50:08 -0400 Subject: [PATCH 35/67] keep analysis status for variant metadata --- seqr/views/apis/report_api_tests.py | 1 + seqr/views/utils/anvil_metadata_utils.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index d532ba142f..ad185e4d6c 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -497,6 +497,7 @@ 'MME': False, 'additional_family_members_with_variant': '', 'allele_balance_or_heteroplasmy_percentage': None, + 'analysisStatus': 'Q', 'clinvar': None, 'condition_id': None, 'end': None, diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 40aca9ca8a..77f0f3b4dc 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -125,6 +125,7 @@ def _get_family_metadata(family_filter, family_fields, include_family_name_displ 'value': ArrayAgg('analysisgroup__name', distinct=True, filter=Q(analysisgroup__isnull=False)), 'format': lambda f: '; '.join(f['analysis_groups']), }} if include_family_sample_metadata else family_fields + include_family_name_display = include_family_name_display or include_family_sample_metadata family_data = Family.objects.filter(**family_filter).distinct().order_by('id').values( 'id', 'family_id', 'post_discovery_omim_numbers', *(['post_discovery_mondo_id'] if include_mondo else []), @@ -135,14 +136,14 @@ def _get_family_metadata(family_filter, family_fields, include_family_name_displ Value('\t'), Value(' '), ), analysisStatus=F('analysis_status'), - **(FAMILY_NAME_DISPLAY_VALUES if include_family_name_display or include_family_sample_metadata else {}), + **(FAMILY_NAME_DISPLAY_VALUES if include_family_name_display else {}), **{k: v['value'] for k, v in (family_fields or {}).items()} ) family_data_by_id = {} for f in family_data: family_id = f.pop('id') - analysis_status = f['analysisStatus'] if include_family_sample_metadata else f.pop('analysisStatus') + analysis_status = f['analysisStatus'] if include_family_name_display else f.pop('analysisStatus') solve_status = ANALYSIS_SOLVE_STATUS_LOOKUP.get(analysis_status, Individual.UNSOLVED) f.update({ 'solve_status': Individual.SOLVE_STATUS_LOOKUP[solve_status], From 21100ed085e688d7997e7244bba508a0c42be864 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 27 Jun 2024 13:41:19 -0400 Subject: [PATCH 36/67] update js tests --- .../components/IndividualMetadata.test.js | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/ui/pages/SummaryData/components/IndividualMetadata.test.js b/ui/pages/SummaryData/components/IndividualMetadata.test.js index abb8c850ae..44c5ad173d 100644 --- a/ui/pages/SummaryData/components/IndividualMetadata.test.js +++ b/ui/pages/SummaryData/components/IndividualMetadata.test.js @@ -14,7 +14,6 @@ const DATA = [ projectGuid: 'R0003_test', num_saved_variants: 2, solve_status: 'Tier 1', - sample_id: 'NA20889', 'gene_known_for_phenotype-1': 'Candidate', 'gene_known_for_phenotype-2': 'Candidate', 'variant_inheritance-1': 'unknown', @@ -31,6 +30,7 @@ const DATA = [ 'sv_name-2': 'DEL:chr12:49045487-49045898', 'chrom-2': '12', 'pos-2': '49045487', + 'end-2': '49045898', maternal_id: '', paternal_id: '', maternal_guid: '', @@ -38,6 +38,7 @@ const DATA = [ 'hgvsp-1': 'c.1586-17C>G', internal_project_id: 'Test Reprocessed Project', 'pos-1': 248367227, + 'end-1': null, data_type: 'WES', familyGuid: 'F000012_12', family_history: 'Yes', @@ -58,10 +59,13 @@ const DATA = [ disorders: null, family_id: '12', displayName: '12', - MME: 'Y', + MME: 'Yes', participant_id: 'NA20889', individual_guid: 'I000017_na20889', proband_relationship: 'Self', + condition_id: 'OMIM:616126', + condition_inheritance: 'Autosomal recessive', + known_condition_name: 'Immunodeficiency 38', 'phenotype_contribution-1': 'Partial', 'phenotype_contribution-2': 'Full', 'partial_contribution_explained-1': 'HP:0000501|HP:0000365', @@ -76,26 +80,26 @@ test('IndividualMetadata render and export', () => { const sampleMetadata = mount() const exportConfig = sampleMetadata.find('DataTable').instance().exportConfig(DATA)[0] expect(exportConfig.headers).toEqual([ - 'project_id', 'projectGuid', 'family_id', 'familyGuid', 'participant_id', 'individual_guid', 'pmid_id', 'paternal_id', - 'paternal_guid', 'maternal_id', 'maternal_guid', 'proband_relationship', 'sex', 'ancestry', - 'condition_id', 'known_condition_name', 'disorders', 'affected_status', 'hpo_present', 'hpo_absent', - 'phenotype_description', 'analysis_groups', 'analysis_status', 'solve_status', 'MME', 'data_type', 'date_data_generation', + 'project_id', 'projectGuid', 'family_id', 'familyGuid', 'participant_id', 'individual_guid', 'paternal_id', + 'paternal_guid', 'maternal_id', 'maternal_guid', 'proband_relationship', 'sex', 'ancestry', 'affected_status', + 'hpo_present', 'hpo_absent', 'MME', 'pmid_id', 'condition_id', 'known_condition_name', 'condition_inheritance', 'disorders', + 'phenotype_description', 'analysis_groups', 'analysis_status', 'solve_status', 'data_type', 'date_data_generation', 'filter_flags', 'consanguinity', 'family_history', 'genetic_findings_id-1', 'variant_reference_assembly-1', - 'chrom-1', 'pos-1', 'ref-1', 'alt-1', 'gene_of_interest-1', 'gene_id-1', 'seqr_chosen_consequence-1', 'transcript-1', + 'chrom-1', 'pos-1', 'end-1', 'ref-1', 'alt-1', 'gene_of_interest-1', 'gene_id-1', 'seqr_chosen_consequence-1', 'transcript-1', 'hgvsc-1', 'hgvsp-1', 'zygosity-1', 'sv_name-1', 'sv_type-1', 'variant_inheritance-1', 'gene_known_for_phenotype-1', 'phenotype_contribution-1', 'partial_contribution_explained-1', 'notes-1', 'ClinGen_allele_ID-1', - 'genetic_findings_id-2', 'variant_reference_assembly-2', 'chrom-2', 'pos-2', + 'genetic_findings_id-2', 'variant_reference_assembly-2', 'chrom-2', 'pos-2', 'end-2', 'ref-2', 'alt-2', 'gene_of_interest-2', 'gene_id-2', 'seqr_chosen_consequence-2', 'transcript-2', 'hgvsc-2', 'hgvsp-2', 'zygosity-2', 'sv_name-2', 'sv_type-2', 'variant_inheritance-2', 'gene_known_for_phenotype-2', 'phenotype_contribution-2', 'partial_contribution_explained-2', 'notes-2', 'ClinGen_allele_ID-2']) expect(exportConfig.processRow(DATA[0])).toEqual([ - 'Test Reprocessed Project', 'R0003_test', '12', 'F000012_12', 'NA20889', 'I000017_na20889', null, '', '', '', '', - 'Self', 'Female', 'Ashkenazi Jewish', undefined, undefined, null, 'Affected', - 'HP:0011675 (Arrhythmia)|HP:0001509 ()', '', null, undefined, 'Waiting for data', 'Tier 1', 'Y', 'WES', '2017-02-05', '', - undefined, 'Yes', 'NA20889_1_248367227', undefined, '1', 248367227, 'TC', 'T', 'OR4G11P', 'ENSG00000240361', - 'intron_variant', 'ENST00000505820', 'c.3955G>A', 'c.1586-17C>G', 'Heterozygous', undefined, undefined, - 'unknown', 'Candidate', 'Partial', 'HP:0000501|HP:0000365', undefined, 'CA1501729', 'NA20889_1_249045487', undefined, '12', '49045487', undefined, - undefined, undefined, undefined, undefined, + 'Test Reprocessed Project', 'R0003_test', '12', 'F000012_12', 'NA20889', 'I000017_na20889', '', '', '', '', + 'Self', 'Female', 'Ashkenazi Jewish', 'Affected', 'HP:0011675 (Arrhythmia)|HP:0001509 ()', '', 'Yes', null, + 'OMIM:616126', 'Immunodeficiency 38', 'Autosomal recessive', null, null, undefined, 'Waiting for data', 'Tier 1', + 'WES', '2017-02-05', '', undefined, 'Yes', 'NA20889_1_248367227', undefined, '1', 248367227, null, 'TC', 'T', + 'OR4G11P', 'ENSG00000240361', 'intron_variant', 'ENST00000505820', 'c.3955G>A', 'c.1586-17C>G', 'Heterozygous', undefined, undefined, + 'unknown', 'Candidate', 'Partial', 'HP:0000501|HP:0000365', undefined, 'CA1501729', 'NA20889_1_249045487', undefined, + '12', '49045487', '49045898', undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, 'Heterozygous', 'DEL:chr12:49045487-49045898', 'Deletion', 'unknown', 'Candidate', 'Full', '', undefined, null]) }) From 49370b85ac168bcb35dbf969d3f232b88e57c3ed Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 27 Jun 2024 14:54:31 -0400 Subject: [PATCH 37/67] fix formatting for saved variant download --- ui/shared/components/panel/variants/selectors.js | 5 +++-- ui/shared/utils/constants.js | 11 ++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/ui/shared/components/panel/variants/selectors.js b/ui/shared/components/panel/variants/selectors.js index a58de15314..651b9975a0 100644 --- a/ui/shared/components/panel/variants/selectors.js +++ b/ui/shared/components/panel/variants/selectors.js @@ -309,10 +309,11 @@ export const getSavedVariantExportConfig = createSelector( getAnalysisGroupsByGuid, getVariantTagsByGuid, getVariantNotesByGuid, + getGenesById, (state, props) => props.project, getSavedVariantTableState, (state, props) => props.match.params, - (analysisGroupsByGuid, tagsByGuid, notesByGuid, project, tableState, params) => { + (analysisGroupsByGuid, tagsByGuid, notesByGuid, genesById, project, tableState, params) => { if (project && project.isDemo && !project.allUserDemo) { // Do not allow downloads for demo projects return null @@ -329,7 +330,7 @@ export const getSavedVariantExportConfig = createSelector( getHeaders: state => getSavedVariantExportHeaders(state, { project, match: { params } }), processRow: variant => ([ ...VARIANT_EXPORT_DATA.map(config => ( - config.getVal ? config.getVal(variant, tagsByGuid, notesByGuid) : variant[config.header])), + config.getVal ? config.getVal(variant, tagsByGuid, notesByGuid, genesById) : variant[config.header])), ...Object.values(variant.genotypes).reduce( (acc, { sampleId, numAlt, gq, ab }) => ([...acc, sampleId, numAlt, gq, ab]), [], ), diff --git a/ui/shared/utils/constants.js b/ui/shared/utils/constants.js index 882b788c1f..9ce1b5f479 100644 --- a/ui/shared/utils/constants.js +++ b/ui/shared/utils/constants.js @@ -1531,14 +1531,19 @@ const getPopAf = population => (variant) => { return (populationData || {}).af } +const getVariantGene = (variant, tagsByGuid, notesByGuid, genesById) => { + const { geneId } = getVariantMainTranscript(variant) + return genesById[geneId]?.geneSymbol || geneId +} + export const VARIANT_EXPORT_DATA = [ { header: 'chrom' }, { header: 'pos' }, { header: 'ref' }, { header: 'alt' }, - { header: 'gene', getVal: variant => getVariantMainTranscript(variant).geneSymbol }, + { header: 'gene', getVal: getVariantGene }, { header: 'worst_consequence', getVal: variant => getVariantMainTranscript(variant).majorConsequence }, - { header: 'callset_freq', getVal: getPopAf('callset') }, + { header: 'callset_freq', getVal: variant => getPopAf('callset')(variant) || getPopAf('seqr')(variant) }, { header: 'exac_freq', getVal: getPopAf('exac') }, { header: 'gnomad_genomes_freq', getVal: getPopAf('gnomad_genomes') }, { header: 'gnomad_exomes_freq', getVal: getPopAf('gnomad_exomes') }, @@ -1554,7 +1559,7 @@ export const VARIANT_EXPORT_DATA = [ { header: 'rsid', getVal: variant => variant.rsid }, { header: 'hgvsc', getVal: variant => getVariantMainTranscript(variant).hgvsc }, { header: 'hgvsp', getVal: variant => getVariantMainTranscript(variant).hgvsp }, - { header: 'clinvar_clinical_significance', getVal: variant => (variant.clinvar || {}).clinicalSignificance }, + { header: 'clinvar_clinical_significance', getVal: variant => (variant.clinvar || {}).clinicalSignificance || (variant.clinvar || {}).pathogenicity }, { header: 'clinvar_gold_stars', getVal: variant => (variant.clinvar || {}).goldStars }, { header: 'filter', getVal: variant => variant.genotypeFilters }, { header: 'project' }, From c1fdc7844e783eb971acee50d071fc4c625f9ee7 Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Thu, 27 Jun 2024 16:22:01 -0400 Subject: [PATCH 38/67] fix individualrow --- ui/pages/Project/components/FamilyTable/IndividualRow.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/pages/Project/components/FamilyTable/IndividualRow.jsx b/ui/pages/Project/components/FamilyTable/IndividualRow.jsx index 8717ebdde3..0d9c035f82 100644 --- a/ui/pages/Project/components/FamilyTable/IndividualRow.jsx +++ b/ui/pages/Project/components/FamilyTable/IndividualRow.jsx @@ -169,7 +169,7 @@ const DataDetails = React.memo(({ loadedSamples, individual, mmeSubmission }) => /> ) : )} - {loadedSamples.some(sample => sample.rnaSeqTypes?.length > 0) && ( + {loadedSamples.some(sample => sample.isActive && (sample.rnaSeqTypes.includes('Expression Outlier') || sample.rnaSeqTypes.includes('Splice Outlier'))) && (
Date: Thu, 27 Jun 2024 16:35:29 -0400 Subject: [PATCH 39/67] allow anvil users to create manual variants --- ui/pages/Project/components/CreateVariantButton.jsx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ui/pages/Project/components/CreateVariantButton.jsx b/ui/pages/Project/components/CreateVariantButton.jsx index 197f2bc491..b0208c856e 100644 --- a/ui/pages/Project/components/CreateVariantButton.jsx +++ b/ui/pages/Project/components/CreateVariantButton.jsx @@ -212,10 +212,11 @@ const SV_FIELDS = [ }, ].map(formatField) -const BaseCreateVariantButton = React.memo(({ variantType, family, user, ...props }) => ( - user.isAnalyst ? ( +const BaseCreateVariantButton = React.memo(({ variantType, family, user, project, ...props }) => ( + (project.isAnalystProject ? user.isAnalyst : project.canEdit) ? ( ({ user: getUser(state), - initialValues: getCurrentProject(state), + project: getCurrentProject(state), }) const mapDispatchToProps = (dispatch, ownProps) => ({ From af4654208f3178c884dcebb7226cff0d80570ae5 Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Thu, 27 Jun 2024 16:43:43 -0400 Subject: [PATCH 40/67] isactive --- ui/shared/components/panel/sample.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/shared/components/panel/sample.jsx b/ui/shared/components/panel/sample.jsx index f19d078a0c..c682c892b4 100644 --- a/ui/shared/components/panel/sample.jsx +++ b/ui/shared/components/panel/sample.jsx @@ -40,7 +40,7 @@ const Sample = React.memo(({ loadedSample, isOutdated, hoverDetails }) => ( 'no data available'} {loadedSample.sampleType && loadedSample.sampleType === SAMPLE_TYPE_RNA && - loadedSample.rnaSeqTypes?.length > 0 && `RNAseq methods: ${loadedSample.rnaSeqTypes.join(', ')}`} + loadedSample.isActive && `RNAseq methods: ${loadedSample.rnaSeqTypes.join(', ')}`}
} position="left center" From 6e0ee86f5ad0cc0f533c4001c45c79be95e7a03c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 28 Jun 2024 11:52:57 -0400 Subject: [PATCH 41/67] add vlm contact db field; --- CHANGELOG.md | 1 + .../0068_project_vlm_contact_email.py | 29 +++++++++++++++++++ seqr/models.py | 5 +++- settings.py | 2 ++ 4 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 seqr/migrations/0068_project_vlm_contact_email.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 984ec30bbb..cf141bec2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # _seqr_ Changes ## dev +* Add VLM contact for Projects (REQUIRES DB MIGRATION) ## 6/11/24 * Add "Partial Phenotype Contribution" functional tag (REQUIRES DB MIGRATION) diff --git a/seqr/migrations/0068_project_vlm_contact_email.py b/seqr/migrations/0068_project_vlm_contact_email.py new file mode 100644 index 0000000000..c158184936 --- /dev/null +++ b/seqr/migrations/0068_project_vlm_contact_email.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.23 on 2024-06-28 15:44 + +from django.db import migrations, models + + +def update_vlm_contact_email(apps, schema_editor): + Project = apps.get_model('seqr', 'Project') + db_alias = schema_editor.connection.alias + + projects = Project.objects.using(db_alias).all() + for project in projects: + project.vlm_contact_email = project.mme_contact_url.replace('mailto:', '').replace('matchmaker', 'vlm') + Project.objects.using(db_alias).bulk_update(projects, ['vlm_contact_email']) + + +class Migration(migrations.Migration): + + dependencies = [ + ('seqr', '0067_alter_variantfunctionaldata_functional_data_tag'), + ] + + operations = [ + migrations.AddField( + model_name='project', + name='vlm_contact_email', + field=models.TextField(blank=True, default='vlm@broadinstitute.org', null=True), + ), + migrations.RunPython(update_vlm_contact_email, reverse_code=migrations.RunPython.noop), + ] diff --git a/seqr/models.py b/seqr/models.py index 1cd1f46b0e..770a37cce1 100644 --- a/seqr/models.py +++ b/seqr/models.py @@ -17,7 +17,8 @@ from seqr.utils.xpos_utils import get_chrom_pos from seqr.views.utils.terra_api_utils import anvil_enabled from reference_data.models import GENOME_VERSION_GRCh37, GENOME_VERSION_CHOICES -from settings import MME_DEFAULT_CONTACT_NAME, MME_DEFAULT_CONTACT_HREF, MME_DEFAULT_CONTACT_INSTITUTION +from settings import MME_DEFAULT_CONTACT_NAME, MME_DEFAULT_CONTACT_HREF, MME_DEFAULT_CONTACT_INSTITUTION, \ + VLM_DEFAULT_CONTACT_EMAIL logger = SeqrLogger(__name__) @@ -196,6 +197,8 @@ class Project(ModelWithGUID): mme_contact_url = models.TextField(null=True, blank=True, default=MME_DEFAULT_CONTACT_HREF) mme_contact_institution = models.TextField(null=True, blank=True, default=MME_DEFAULT_CONTACT_INSTITUTION) + vlm_contact_email = models.TextField(null=True, blank=True, default=VLM_DEFAULT_CONTACT_EMAIL) + has_case_review = models.BooleanField(default=False) enable_hgmd = models.BooleanField(default=False) all_user_demo = models.BooleanField(default=False) diff --git a/settings.py b/settings.py index b7fd626bfa..852d1ddac4 100644 --- a/settings.py +++ b/settings.py @@ -352,6 +352,8 @@ MME_DEFAULT_CONTACT_EMAIL = 'matchmaker@broadinstitute.org' MME_DEFAULT_CONTACT_HREF = 'mailto:{}'.format(MME_DEFAULT_CONTACT_EMAIL) +VLM_DEFAULT_CONTACT_EMAIL = 'vlm@broadinstitute.org' + MME_CONFIG_DIR = os.environ.get('MME_CONFIG_DIR', '') MME_NODES = {} if MME_CONFIG_DIR: From d1c6891ed4953c8bb0322f996a08a79d259f0950 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 28 Jun 2024 12:10:11 -0400 Subject: [PATCH 42/67] use vlm_contact_email in create and responses --- seqr/fixtures/1kg_project.json | 4 +++ seqr/models.py | 1 + seqr/views/apis/anvil_workspace_api.py | 1 + seqr/views/apis/anvil_workspace_api_tests.py | 29 ++++++++++++++------ seqr/views/apis/project_api_tests.py | 27 ++++++++++++++---- seqr/views/utils/test_utils.py | 2 +- 6 files changed, 49 insertions(+), 15 deletions(-) diff --git a/seqr/fixtures/1kg_project.json b/seqr/fixtures/1kg_project.json index bad151c995..6f83fda996 100644 --- a/seqr/fixtures/1kg_project.json +++ b/seqr/fixtures/1kg_project.json @@ -20,6 +20,7 @@ "has_case_review": true, "mme_primary_data_owner": "PI", "mme_contact_url": "mailto:test@broadinstitute.org,matchmaker@broadinstitute.org", + "vlm_contact_email": "test@broadinstitute.org,vlm@broadinstitute.org", "last_accessed_date": "2017-09-15T18:15:50.827Z" } }, @@ -41,6 +42,7 @@ "can_view_group": 3, "is_mme_enabled": false, "mme_primary_data_owner": "", + "vlm_contact_email": "vlm@broadinstitute.org", "last_accessed_date": "2017-09-15T18:15:50.827Z" } }, @@ -63,6 +65,7 @@ "is_demo": true, "mme_primary_data_owner": "", "mme_contact_url": "mailto:seqr-test@gmail.com,test@broadinstitute.org", + "vlm_contact_email": "seqr-test@gmail.com,test@broadinstitute.org", "last_accessed_date": "2017-09-15T18:15:50.827Z" } }, @@ -81,6 +84,7 @@ "last_accessed_date": "2017-09-15T18:15:50.827Z", "consent_code": "H", "genome_version": "38", + "vlm_contact_email": "vlm@broadinstitute.org", "workspace_name": "anvil-non-analyst-project 1000 Genomes Demo", "workspace_namespace": "ext-data" } diff --git a/seqr/models.py b/seqr/models.py index 770a37cce1..bda633b9b7 100644 --- a/seqr/models.py +++ b/seqr/models.py @@ -264,6 +264,7 @@ class Meta: 'name', 'description', 'created_date', 'last_modified_date', 'genome_version', 'mme_contact_institution', 'last_accessed_date', 'is_mme_enabled', 'mme_primary_data_owner', 'mme_contact_url', 'guid', 'consent_code', 'workspace_namespace', 'workspace_name', 'has_case_review', 'enable_hgmd', 'is_demo', 'all_user_demo', + 'vlm_contact_email', ] diff --git a/seqr/views/apis/anvil_workspace_api.py b/seqr/views/apis/anvil_workspace_api.py index 281be65beb..9a0f006c14 100644 --- a/seqr/views/apis/anvil_workspace_api.py +++ b/seqr/views/apis/anvil_workspace_api.py @@ -184,6 +184,7 @@ def create_project_from_workspace(request, namespace, name): 'workspace_name': name, 'mme_primary_data_owner': request.user.get_full_name(), 'mme_contact_url': 'mailto:{}'.format(request.user.email), + 'vlm_contact_email': request.user.email, } project = create_model_from_json(Project, project_args, user=request.user) diff --git a/seqr/views/apis/anvil_workspace_api_tests.py b/seqr/views/apis/anvil_workspace_api_tests.py index f24cc485d1..4f000b641b 100644 --- a/seqr/views/apis/anvil_workspace_api_tests.py +++ b/seqr/views/apis/anvil_workspace_api_tests.py @@ -559,14 +559,27 @@ def test_create_project_from_workspace(self): self.assertEqual(response.status_code, 200) project = Project.objects.get(workspace_namespace=TEST_WORKSPACE_NAMESPACE, workspace_name=TEST_NO_PROJECT_WORKSPACE_NAME) response_json = response.json() - self.assertEqual(project.guid, response_json['projectGuid']) - self.assertListEqual( - [project.genome_version, project.description, project.workspace_namespace, project.workspace_name], - ['38', 'A test project', TEST_WORKSPACE_NAMESPACE, TEST_NO_PROJECT_WORKSPACE_NAME]) - - self.assertListEqual( - [project.mme_contact_institution, project.mme_primary_data_owner, project.mme_contact_url], - ['Broad Center for Mendelian Genomics', 'Test Manager User', 'mailto:test_user_manager@test.com']) + self.assertDictEqual({k: getattr(project, k) for k in project._meta.json_fields}, { + 'guid': response_json['projectGuid'], + 'name': TEST_NO_PROJECT_WORKSPACE_NAME, + 'description': 'A test project', + 'workspace_namespace': TEST_WORKSPACE_NAMESPACE, + 'workspace_name': TEST_NO_PROJECT_WORKSPACE_NAME, + 'has_case_review': False, + 'enable_hgmd': False, + 'is_demo': False, + 'all_user_demo': False, + 'consent_code': None, + 'created_date': mock.ANY, + 'last_modified_date': mock.ANY, + 'last_accessed_date': mock.ANY, + 'genome_version': '38', + 'is_mme_enabled': True, + 'mme_contact_institution': 'Broad Center for Mendelian Genomics', + 'mme_primary_data_owner': 'Test Manager User', + 'mme_contact_url': 'mailto:test_user_manager@test.com', + 'vlm_contact_email': 'test_user_manager@test.com', + }) self._assert_valid_operation(project, test_add_data=False) diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py index 3f07dd8526..7c5c821542 100644 --- a/seqr/views/apis/project_api_tests.py +++ b/seqr/views/apis/project_api_tests.py @@ -73,15 +73,30 @@ def test_create_and_delete_project(self, mock_airtable_logger): # check that project was created new_project = Project.objects.get(name='new_project') - self.assertEqual(new_project.description, 'new project description') - self.assertEqual(new_project.genome_version, '38') - self.assertEqual(new_project.consent_code, 'H') - self.assertTrue(new_project.is_demo) - self.assertFalse(new_project.is_mme_enabled) self.assertEqual(new_project.created_by, self.pm_user) self.assertEqual(new_project.projectcategory_set.count(), 0) expected_workspace_name = self.CREATE_PROJECT_JSON.get('workspaceName') - self.assertEqual(new_project.workspace_name, expected_workspace_name) + self.assertDictEqual({k: getattr(new_project, k) for k in new_project._meta.json_fields}, { + 'guid': mock.ANY, + 'name': 'new_project', + 'description': 'new project description', + 'workspace_namespace': self.CREATE_PROJECT_JSON.get('workspaceNamespace'), + 'workspace_name': expected_workspace_name, + 'has_case_review': False, + 'enable_hgmd': False, + 'is_demo': True, + 'all_user_demo': False, + 'consent_code': 'H', + 'created_date': mock.ANY, + 'last_modified_date': mock.ANY, + 'last_accessed_date': mock.ANY, + 'genome_version': '38', + 'is_mme_enabled': False, + 'mme_contact_institution': 'Broad Center for Mendelian Genomics', + 'mme_primary_data_owner': 'Samantha Baxter', + 'mme_contact_url': 'mailto:matchmaker@broadinstitute.org', + 'vlm_contact_email': 'vlm@broadinstitute.org', + }) self._check_created_project_groups(new_project) project_guid = new_project.guid diff --git a/seqr/views/utils/test_utils.py b/seqr/views/utils/test_utils.py index f0ed234d4f..7ff2f6e935 100644 --- a/seqr/views/utils/test_utils.py +++ b/seqr/views/utils/test_utils.py @@ -738,7 +738,7 @@ def _get_list_param(call, param): 'projectGuid', 'projectCategoryGuids', 'canEdit', 'name', 'description', 'createdDate', 'lastModifiedDate', 'lastAccessedDate', 'mmeContactUrl', 'genomeVersion', 'mmePrimaryDataOwner', 'mmeContactInstitution', 'isMmeEnabled', 'workspaceName', 'workspaceNamespace', 'hasCaseReview', 'enableHgmd', 'isDemo', 'allUserDemo', - 'userIsCreator', 'consentCode', 'isAnalystProject', + 'userIsCreator', 'consentCode', 'isAnalystProject', 'vlmContactEmail', } ANALYSIS_GROUP_FIELDS = {'analysisGroupGuid', 'description', 'name', 'projectGuid', 'familyGuids'} From 0c6a3d1b43791f058e90322dac5102a6211191c9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 28 Jun 2024 13:20:14 -0400 Subject: [PATCH 43/67] edit vlm cotnact --- .../components/buttons/EditProjectButton.jsx | 45 ++++++++++++++++++- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/ui/shared/components/buttons/EditProjectButton.jsx b/ui/shared/components/buttons/EditProjectButton.jsx index f1ca2212b4..3c3bedee63 100644 --- a/ui/shared/components/buttons/EditProjectButton.jsx +++ b/ui/shared/components/buttons/EditProjectButton.jsx @@ -3,6 +3,8 @@ import { connect } from 'react-redux' import PropTypes from 'prop-types' import { updateProject } from 'redux/rootReducer' +import { BaseSemanticInput } from '../form/Inputs' +import { validators } from '../form/FormHelpers' import UpdateButton from './UpdateButton' import { EDITABLE_PROJECT_FIELDS, @@ -11,10 +13,49 @@ import { MATCHMAKER_CONTACT_URL_FIELD, } from '../../utils/constants' -const MATCHMAKER_PROJECT_FIELDS = [ +const setBoolVal = onChange => data => onChange(data.checked ? null : 'vlm@broadinstitute.org') + +const VlmContactInput = ({ value, onChange, ...props }) => ([ + , + , +]) + +VlmContactInput.propTypes = { + value: PropTypes.string, + onChange: PropTypes.func, +} + +const VLM_CONTACT_FIELD = { + name: 'vlmContactEmail', + parse: val => val || null, + format: val => val || '', + validate: value => (!value ? undefined : validators.requiredEmail(value)), + component: VlmContactInput, +} + +const MATCHMAKER_PROJECT_FIELDS = [VLM_CONTACT_FIELD, ...[ { ...MATCHMAKER_CONTACT_NAME_FIELD, name: 'mmePrimaryDataOwner' }, { ...MATCHMAKER_CONTACT_URL_FIELD, name: 'mmeContactUrl' }, -].map(({ label, ...field }) => ({ ...field, label: `Matchmaker ${label}` })) +].map(({ label, ...field }) => ({ ...field, label: `Matchmaker ${label}` }))] // Field mapping based on whether project has matchmaker and user is a PM. Usage: FIELD_LOOKUP[isMmeEnabled][isPm] const FIELD_LOOKUP = { From 1433dedf01dde2e4f88b9a2a4356f121a72aa2b7 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 1 Jul 2024 10:58:01 -0400 Subject: [PATCH 44/67] actually test gregor finding row with notes --- seqr/views/apis/individual_api_tests.py | 14 +++++++------- seqr/views/apis/report_api_tests.py | 10 +++++++--- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index 079275b04a..72f72df621 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -994,7 +994,7 @@ def _set_metadata_file_iter(self, mock_subprocess, genetic_findings_table): @mock.patch('seqr.utils.file_utils.subprocess.Popen') def test_import_gregor_metadata(self, mock_subprocess): genetic_findings_table = deepcopy(GENETIC_FINDINGS_TABLE) - genetic_findings_table[2] = genetic_findings_table[2][:11] + genetic_findings_table[3][11:14] + \ + genetic_findings_table[2] = genetic_findings_table[2][:11] + genetic_findings_table[4][11:14] + \ genetic_findings_table[2][14:] self._set_metadata_file_iter(mock_subprocess, genetic_findings_table) @@ -1021,7 +1021,7 @@ def test_import_gregor_metadata(self, mock_subprocess): 'Created 1 new families, 3 new individuals', 'Updated 1 existing families, 1 existing individuals', 'Skipped 0 unchanged individuals', - 'Loaded 3 new and 0 updated findings tags', + 'Loaded 4 new and 0 updated findings tags', ], }}) @@ -1036,7 +1036,7 @@ def test_import_gregor_metadata(self, mock_subprocess): 'metadataTitle': None, 'color': '#c25fc4', 'order': 0.5, - 'numTags': 4, + 'numTags': 5, }) self.assertEqual(len(response_json['familiesByGuid']), 2) @@ -1047,7 +1047,7 @@ def test_import_gregor_metadata(self, mock_subprocess): self.assertDictEqual(response_json['familyTagTypeCounts'], { 'F000012_12': {'GREGoR Finding': 3, 'MME Submission': 2, 'Tier 1 - Novel gene and phenotype': 1}, - new_family_guid: {'GREGoR Finding': 1}, + new_family_guid: {'GREGoR Finding': 2}, }) self.assertEqual(len(response_json['individualsByGuid']), 4) @@ -1126,7 +1126,7 @@ def test_import_gregor_metadata(self, mock_subprocess): 'saved_variant_json__transcripts', 'saved_variant_json__genotypes', 'saved_variant_json__mainTranscriptId', 'saved_variant_json__hgvsc', ) - self.assertEqual(len(saved_variants), 3) + self.assertEqual(len(saved_variants), 4) self.assertDictEqual(saved_variants[0], { 'guid': 'SV0000006_1248367227_r0003_tes', 'variant_id': '1-248367227-TC-T', @@ -1220,12 +1220,12 @@ def test_import_gregor_metadata(self, mock_subprocess): 'Created 0 new families, 0 new individuals', 'Updated 0 existing families, 0 existing individuals', 'Skipped 4 unchanged individuals', - 'Loaded 1 new and 2 updated findings tags', + 'Loaded 1 new and 3 updated findings tags', ], }}) self.assertDictEqual(response_json['individualsByGuid'], {}) - no_gene_saved_variant_json = SavedVariant.objects.get(family__guid=new_family_guid).saved_variant_json + no_gene_saved_variant_json = SavedVariant.objects.get(family__guid=new_family_guid, variant_id='1-248367227-TC-T').saved_variant_json self.assertDictEqual(no_gene_saved_variant_json['transcripts'], {}) self.assertDictEqual(no_gene_saved_variant_json['genotypes'], new_family_genotypes) self.assertNotIn('mainTranscriptId', no_gene_saved_variant_json) diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index ad185e4d6c..a304da2407 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -608,6 +608,10 @@ 'Broad_HG00731_1_248367227', 'Broad_HG00731', 'Broad_exome_VCGS_FAM203_621_D2', 'SNV/INDEL', 'GRCh37', '1', '248367227', 'TC', 'T', 'CA1501729', 'RP11', '', '', '', 'Homozygous', '', 'paternal', '', '', 'Known', '', 'MONDO:0044970', '', 'Uncertain', '', 'Broad_HG00732', 'SR-ES', '', '', '', '', '', '', '', + ], [ + 'Broad_HG00731_19_1912634', 'Broad_HG00731', 'Broad_exome_VCGS_FAM203_621_D2', 'SNV/INDEL', 'GRCh38', '19', + '1912634', 'C', 'T', 'CA403171634', 'OR4G11P', 'ENST00000371839', '', '', 'Heterozygous', '', 'unknown', + 'Broad_HG00731_19_1912633', '', 'Known', '', 'MONDO:0044970', '', 'Full', '', '', 'SR-ES', '', '', '', '', '', '', '', ], [ 'Broad_NA20889_1_248367227', 'Broad_NA20889', '', 'SNV/INDEL', 'GRCh37', '1', '248367227', 'TC', 'T', 'CA1501729', 'OR4G11P', 'ENST00000505820', 'c.3955G>A', 'c.1586-17C>G', 'Heterozygous', '', 'unknown', @@ -1100,9 +1104,9 @@ def _assert_expected_gregor_files(self, mock_open, mock_subprocess, has_second_p self._assert_expected_file( genetic_findings_file, - expected_rows=GENETIC_FINDINGS_TABLE if has_second_project else GENETIC_FINDINGS_TABLE[:3], - absent_rows=None if has_second_project else EXPERIMENT_LOOKUP_TABLE[3:], - additional_calls=3, + expected_rows=GENETIC_FINDINGS_TABLE if has_second_project else GENETIC_FINDINGS_TABLE[:4], + absent_rows=None, + additional_calls=2, ) def _assert_expected_file(self, actual_rows, expected_rows, additional_calls=0, absent_rows=None): From b0678e866c60fda53d14ca0505f4a12b1d277690 Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Mon, 1 Jul 2024 12:07:35 -0400 Subject: [PATCH 45/67] static files --- requirements-dev.in | 10 ++-- requirements-dev.txt | 12 ++--- requirements.in | 4 +- requirements.txt | 6 ++- .../static}/fonts/icon-overrides.eot | Bin .../static}/fonts/icon-overrides.svg | 0 .../static}/fonts/icon-overrides.ttf | Bin .../static}/fonts/icon-overrides.woff | Bin .../static}/images/landing_page_icon1.png | Bin .../static}/images/landing_page_icon2.png | Bin .../static}/images/landing_page_icon3.png | Bin .../static}/images/table_excel.png | Bin {static => seqr/static}/images/table_tsv.png | Bin settings.py | 48 +++++++++--------- 14 files changed, 37 insertions(+), 43 deletions(-) rename {static => seqr/static}/fonts/icon-overrides.eot (100%) rename {static => seqr/static}/fonts/icon-overrides.svg (100%) rename {static => seqr/static}/fonts/icon-overrides.ttf (100%) rename {static => seqr/static}/fonts/icon-overrides.woff (100%) rename {static => seqr/static}/images/landing_page_icon1.png (100%) rename {static => seqr/static}/images/landing_page_icon2.png (100%) rename {static => seqr/static}/images/landing_page_icon3.png (100%) rename {static => seqr/static}/images/table_excel.png (100%) rename {static => seqr/static}/images/table_tsv.png (100%) diff --git a/requirements-dev.in b/requirements-dev.in index ff056bf0d8..7d689c6013 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -1,8 +1,8 @@ -c requirements.txt # use the generated reqs as a constraint coverage<5.2 django-compressor -django-debug-toolbar<3.3 # https://github.com/jazzband/django-debug-toolbar -mock # mock objects for unit tests -pip-tools # tool for managing our python dependency tree -responses # mock HTTP responses for unit tests -urllib3-mock # mock urllib3 for tests +django-debug-toolbar==4.4.2 # https://github.com/jazzband/django-debug-toolbar +mock # mock objects for unit tests +pip-tools # tool for managing our python dependency tree +responses # mock HTTP responses for unit tests +urllib3-mock # mock urllib3 for tests diff --git a/requirements-dev.txt b/requirements-dev.txt index 38ec6ac2fc..a1399355e1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -22,7 +22,7 @@ click==8.1.3 # via pip-tools coverage==5.1 # via -r requirements-dev.in -django==3.2.25 +django==4.2.13 # via # -c requirements.txt # django-appconf @@ -31,7 +31,7 @@ django-appconf==1.0.5 # via django-compressor django-compressor==4.3.1 # via -r requirements-dev.in -django-debug-toolbar==3.2.4 +django-debug-toolbar==4.4.2 # via -r requirements-dev.in idna==3.7 # via @@ -47,10 +47,6 @@ pip-tools==6.12.2 # via -r requirements-dev.in pyproject-hooks==1.0.0 # via build -pytz==2022.7.1 - # via - # -c requirements.txt - # django rcssmin==1.1.1 # via django-compressor requests==2.32.2 @@ -69,9 +65,7 @@ sqlparse==0.5.0 toml==0.10.2 # via responses tomli==2.0.1 - # via - # build - # pyproject-hooks + # via build types-toml==0.10.8.5 # via responses urllib3==1.26.19 diff --git a/requirements.in b/requirements.in index 4a76de1a1f..602875d970 100644 --- a/requirements.in +++ b/requirements.in @@ -1,4 +1,4 @@ -Django==4.2 # core server-side framework +Django==4.2.13 # core server-side framework django-anymail # for sending emails using cloud-based mail service providers django-csp # for setting CSP headers django-guardian # object-level permissions for database records. Behind a major version due to missing Python 2 support @@ -14,7 +14,7 @@ gunicorn # web server jmespath openpyxl # library for reading/writing Excel files pillow # required dependency of Djagno ImageField-type database records -psycopg2 # postgres database access +psycopg # postgres database access pyliftover # GRCh37/GRCh38 liftover requests # simpler way to make http requests redis<4.6 # client lib for the redis in-memory database - used for caching server-side objects diff --git a/requirements.txt b/requirements.txt index 83322e1a7a..f099f1b249 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,7 @@ defusedxml==0.7.1 # via # python3-openid # social-auth-core -django==4.2 +django==4.2.13 # via # -r requirements.in # django-anymail @@ -107,7 +107,7 @@ protobuf==3.20.2 # via # google-api-core # googleapis-common-protos -psycopg2==2.9.5 +psycopg==3.2.1 # via -r requirements.in pyasn1==0.4.8 # via @@ -173,6 +173,8 @@ tenacity==8.3.0 # via -r requirements.in tqdm==4.66.3 # via -r requirements.in +typing-extensions==4.12.2 + # via psycopg urllib3==1.26.19 # via # elasticsearch diff --git a/static/fonts/icon-overrides.eot b/seqr/static/fonts/icon-overrides.eot similarity index 100% rename from static/fonts/icon-overrides.eot rename to seqr/static/fonts/icon-overrides.eot diff --git a/static/fonts/icon-overrides.svg b/seqr/static/fonts/icon-overrides.svg similarity index 100% rename from static/fonts/icon-overrides.svg rename to seqr/static/fonts/icon-overrides.svg diff --git a/static/fonts/icon-overrides.ttf b/seqr/static/fonts/icon-overrides.ttf similarity index 100% rename from static/fonts/icon-overrides.ttf rename to seqr/static/fonts/icon-overrides.ttf diff --git a/static/fonts/icon-overrides.woff b/seqr/static/fonts/icon-overrides.woff similarity index 100% rename from static/fonts/icon-overrides.woff rename to seqr/static/fonts/icon-overrides.woff diff --git a/static/images/landing_page_icon1.png b/seqr/static/images/landing_page_icon1.png similarity index 100% rename from static/images/landing_page_icon1.png rename to seqr/static/images/landing_page_icon1.png diff --git a/static/images/landing_page_icon2.png b/seqr/static/images/landing_page_icon2.png similarity index 100% rename from static/images/landing_page_icon2.png rename to seqr/static/images/landing_page_icon2.png diff --git a/static/images/landing_page_icon3.png b/seqr/static/images/landing_page_icon3.png similarity index 100% rename from static/images/landing_page_icon3.png rename to seqr/static/images/landing_page_icon3.png diff --git a/static/images/table_excel.png b/seqr/static/images/table_excel.png similarity index 100% rename from static/images/table_excel.png rename to seqr/static/images/table_excel.png diff --git a/static/images/table_tsv.png b/seqr/static/images/table_tsv.png similarity index 100% rename from static/images/table_tsv.png rename to seqr/static/images/table_tsv.png diff --git a/settings.py b/settings.py index 5598d9dadd..2080bf7a8d 100644 --- a/settings.py +++ b/settings.py @@ -2,7 +2,7 @@ import os import random import string -import subprocess # nosec +import subprocess # nosec from ssl import create_default_context @@ -16,7 +16,7 @@ # Django settings ######################################################### -# Password validation - https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators +# Password validation - https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ { 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', @@ -71,7 +71,7 @@ CSRF_COOKIE_NAME = 'csrf_token' CSRF_COOKIE_HTTPONLY = False -SESSION_COOKIE_AGE = 86400 # seconds in 1 day +SESSION_COOKIE_AGE = 86400 # seconds in 1 day X_FRAME_OPTIONS = 'SAMEORIGIN' SECURE_BROWSER_XSS_FILTER = True @@ -127,10 +127,28 @@ USE_L10N = True USE_TZ = True +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/4.2/howto/static-files/ +STATIC_URL = '/static/' +STATICFILES_DIRS = ['ui/dist'] +STATIC_ROOT = os.path.join(BASE_DIR, 'static') +STATICFILES_FINDERS = ( + 'django.contrib.staticfiles.finders.FileSystemFinder', + 'django.contrib.staticfiles.finders.AppDirectoriesFinder', +) +STORAGES = { + "staticfiles": { + "BACKEND": "whitenoise.storage.CompressedManifestStaticFilesStorage", + }, +} + # If specified, store data in the named GCS bucket and use the gcloud storage backend. # Else, fall back to a path on the local filesystem. GCS_MEDIA_ROOT_BUCKET = os.environ.get('GCS_MEDIA_ROOT_BUCKET') if GCS_MEDIA_ROOT_BUCKET: + STORAGES['default'] = { + "BACKEND": "storages.backends.gcloud.GoogleCloudStorage", + } DEFAULT_FILE_STORAGE = 'storages.backends.gcloud.GoogleCloudStorage' GS_BUCKET_NAME = GCS_MEDIA_ROOT_BUCKET GS_DEFAULT_ACL = 'publicRead' @@ -209,7 +227,7 @@ LOGOUT_URL = '/logout' POSTGRES_DB_CONFIG = { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'ENGINE': 'django.db.backends.postgresql', 'HOST': os.environ.get('POSTGRES_SERVICE_HOSTNAME', 'localhost'), 'PORT': int(os.environ.get('POSTGRES_SERVICE_PORT', '5432')), 'USER': os.environ.get('POSTGRES_USERNAME', 'postgres'), @@ -252,10 +270,6 @@ 'http://localhost:3000', 'http://localhost:8000', ) - # TODO: ? - # the collectstatic step in docker build runs without env variables set, and uncommenting these lines breaks the docker build - # STATICFILES_DIRS.append(STATIC_ROOT) - # STATIC_ROOT = None CORS_ALLOW_CREDENTIALS = True CORS_REPLACE_HTTPS_REFERER = True # django-hijack plugin @@ -264,22 +278,6 @@ HIJACK_LOGIN_REDIRECT_URL = '/' TEMPLATE_DIRS.append('ui') - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/4.2/howto/static-files/ -STATIC_URL = '/static/' -STATICFILES_DIRS = ['ui/dist'] -if DEBUG: - STATICFILES_DIRS.append(os.path.join(BASE_DIR, 'static')) -else: - STATIC_ROOT = os.path.join(BASE_DIR, 'static') - -STATICFILES_FINDERS = ( - 'django.contrib.staticfiles.finders.FileSystemFinder', - 'django.contrib.staticfiles.finders.AppDirectoriesFinder', -) - - TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', @@ -289,7 +287,7 @@ 'context_processors': [ 'django.contrib.auth.context_processors.auth', 'django.contrib.messages.context_processors.messages', # required for admin template - 'django.template.context_processors.request', # must be enabled in DjangoTemplates (TEMPLATES) in order to use the admin navigation sidebar + 'django.template.context_processors.request', # must be enabled in DjangoTemplates (TEMPLATES) in order to use the admin navigation sidebar 'social_django.context_processors.backends', # required for social_auth, same for below 'social_django.context_processors.login_redirect', ], From 26e0ff3eb5522ea059dbdd975f7ec15e3379b85d Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Mon, 1 Jul 2024 12:27:09 -0400 Subject: [PATCH 46/67] outputfield --- seqr/migrations/0024_varianttag_metadata.py | 7 ++++++- settings.py | 1 - 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/seqr/migrations/0024_varianttag_metadata.py b/seqr/migrations/0024_varianttag_metadata.py index 8031a4c29f..e522320cd4 100644 --- a/seqr/migrations/0024_varianttag_metadata.py +++ b/seqr/migrations/0024_varianttag_metadata.py @@ -2,6 +2,7 @@ from collections import defaultdict from django.contrib.postgres.aggregates import StringAgg from django.db import migrations, models +from django.db.models import TextField from django.db.models.functions import Concat from django.utils import timezone from seqr.utils.logging_utils import log_model_update, log_model_bulk_update, SeqrLogger @@ -120,7 +121,11 @@ def merge_duplicate_tags(apps, schema_editor): db_alias = schema_editor.connection.alias updated_tags = VariantTag.objects.using(db_alias).filter(variant_tag_type__name__in=SANGER_TAGS.values()).annotate( - group_id=Concat('variant_tag_type__guid', StringAgg('saved_variants__guid', ',', ordering='saved_variants__guid'))) + group_id=Concat( + 'variant_tag_type__guid', + StringAgg('saved_variants__guid', ',', ordering='saved_variants__guid'), + output_field=TextField() + )) if not updated_tags: logger.info('No updated tags found, skipping validation tag merging', user=None) return diff --git a/settings.py b/settings.py index 2080bf7a8d..a29165845c 100644 --- a/settings.py +++ b/settings.py @@ -149,7 +149,6 @@ STORAGES['default'] = { "BACKEND": "storages.backends.gcloud.GoogleCloudStorage", } - DEFAULT_FILE_STORAGE = 'storages.backends.gcloud.GoogleCloudStorage' GS_BUCKET_NAME = GCS_MEDIA_ROOT_BUCKET GS_DEFAULT_ACL = 'publicRead' MEDIA_ROOT = False From ba90486ffeda07bb3fab94d3629114093f460d42 Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Mon, 1 Jul 2024 12:48:00 -0400 Subject: [PATCH 47/67] fix terra api tests --- seqr/views/utils/terra_api_utils.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/seqr/views/utils/terra_api_utils.py b/seqr/views/utils/terra_api_utils.py index 4a81f15c18..dbda82aee8 100644 --- a/seqr/views/utils/terra_api_utils.py +++ b/seqr/views/utils/terra_api_utils.py @@ -73,8 +73,10 @@ def is_google_authenticated(user): def remove_token(user): social = _safe_get_social(user) if social and social.extra_data: - social.extra_data.pop('access_token', None) - social.extra_data['expires'] = 0 + extra_data = json.loads(social.extra_data) + extra_data.pop('access_token', None) + extra_data['expires'] = 0 + social.extra_data = extra_data social.save() @@ -84,7 +86,7 @@ def is_anvil_authenticated(user): social = _safe_get_social(user) if social and social.extra_data: - return social.extra_data.get('access_token', '') != '' + return json.loads(social.extra_data).get('access_token', '') != '' return False @@ -109,7 +111,8 @@ def _safe_get_social(user): def _get_social_access_token(user): social = _safe_get_social(user) - if (social.extra_data['auth_time'] + social.extra_data['expires'] - 10) <= int( + extra_data = json.loads(social.extra_data) + if (extra_data['auth_time'] + extra_data['expires'] - 10) <= int( time.time()): # token expired or expiring? strategy = load_strategy() logger.info('Refreshing access token', user) @@ -118,7 +121,7 @@ def _get_social_access_token(user): except Exception as ee: logger.warning('Refresh token failed. {}'.format(str(ee)), user) raise TerraRefreshTokenFailedException('Refresh token failed. {}'.format(str(ee))) - return social.extra_data['access_token'] + return extra_data['access_token'] def _get_service_account_access_token(): From 288db1293e537cb8ca587439d33bbf35fcd6bd41 Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Mon, 1 Jul 2024 12:53:04 -0400 Subject: [PATCH 48/67] fix superuser tests --- settings.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/settings.py b/settings.py index a29165845c..a3e7e502f5 100644 --- a/settings.py +++ b/settings.py @@ -136,18 +136,15 @@ 'django.contrib.staticfiles.finders.FileSystemFinder', 'django.contrib.staticfiles.finders.AppDirectoriesFinder', ) -STORAGES = { - "staticfiles": { - "BACKEND": "whitenoise.storage.CompressedManifestStaticFilesStorage", - }, -} # If specified, store data in the named GCS bucket and use the gcloud storage backend. # Else, fall back to a path on the local filesystem. GCS_MEDIA_ROOT_BUCKET = os.environ.get('GCS_MEDIA_ROOT_BUCKET') if GCS_MEDIA_ROOT_BUCKET: - STORAGES['default'] = { - "BACKEND": "storages.backends.gcloud.GoogleCloudStorage", + STORAGES = { + "default": { + "BACKEND": "storages.backends.gcloud.GoogleCloudStorage", + } } GS_BUCKET_NAME = GCS_MEDIA_ROOT_BUCKET GS_DEFAULT_ACL = 'publicRead' From 6bbf0b89dc8efbf3593d79d2cb26f64d4de1ee80 Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Mon, 1 Jul 2024 13:28:08 -0400 Subject: [PATCH 49/67] fix social a better way --- seqr/views/utils/terra_api_utils.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/seqr/views/utils/terra_api_utils.py b/seqr/views/utils/terra_api_utils.py index dbda82aee8..46d7ab80e7 100644 --- a/seqr/views/utils/terra_api_utils.py +++ b/seqr/views/utils/terra_api_utils.py @@ -73,10 +73,8 @@ def is_google_authenticated(user): def remove_token(user): social = _safe_get_social(user) if social and social.extra_data: - extra_data = json.loads(social.extra_data) - extra_data.pop('access_token', None) - extra_data['expires'] = 0 - social.extra_data = extra_data + social.extra_data.pop('access_token', None) + social.extra_data['expires'] = 0 social.save() @@ -86,7 +84,7 @@ def is_anvil_authenticated(user): social = _safe_get_social(user) if social and social.extra_data: - return json.loads(social.extra_data).get('access_token', '') != '' + return social.extra_data.get('access_token', '') != '' return False @@ -105,14 +103,19 @@ def _safe_get_social(user): if not google_auth_enabled() or not hasattr(user, 'social_auth'): return None - social = user.social_auth.filter(provider=SOCIAL_AUTH_PROVIDER) - return social.first() if social else None + social_auth = user.social_auth.filter(provider=SOCIAL_AUTH_PROVIDER) + if not social_auth: + return None + + social = social_auth.first() + if type(social.extra_data) is str: # JSONField extra_data is returned as a string + social.extra_data = json.loads(social.extra_data) + return social def _get_social_access_token(user): social = _safe_get_social(user) - extra_data = json.loads(social.extra_data) - if (extra_data['auth_time'] + extra_data['expires'] - 10) <= int( + if (social.extra_data['auth_time'] + social.extra_data['expires'] - 10) <= int( time.time()): # token expired or expiring? strategy = load_strategy() logger.info('Refreshing access token', user) @@ -121,7 +124,7 @@ def _get_social_access_token(user): except Exception as ee: logger.warning('Refresh token failed. {}'.format(str(ee)), user) raise TerraRefreshTokenFailedException('Refresh token failed. {}'.format(str(ee))) - return extra_data['access_token'] + return social.extra_data['access_token'] def _get_service_account_access_token(): From 6891bf4f542ea04aa27ae01b5d0a6b91fdcde39f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 1 Jul 2024 13:59:21 -0400 Subject: [PATCH 50/67] include notes in gregor export --- seqr/views/apis/report_api.py | 3 +- seqr/views/apis/report_api_tests.py | 9 +++-- seqr/views/utils/anvil_metadata_utils.py | 50 +++++++++++++----------- 3 files changed, 35 insertions(+), 27 deletions(-) diff --git a/seqr/views/apis/report_api.py b/seqr/views/apis/report_api.py index 6a6948a0fb..12382aae14 100644 --- a/seqr/views/apis/report_api.py +++ b/seqr/views/apis/report_api.py @@ -400,6 +400,7 @@ def _add_row(row, family_id, row_type): format_id=_format_gregor_id, get_additional_individual_fields=_get_participant_row, post_process_variant=_post_process_gregor_variant, + include_parent_mnvs=True, # TODO flag needed? include_svs=False, airtable_fields=[[PARTICIPANT_ID_FIELD, 'Recontactable'], [SMID_FIELD]], include_mondo=True, @@ -576,7 +577,7 @@ def _get_phenotype_row(feature): } -def _post_process_gregor_variant(row, gene_variants, **kwargs): +def _post_process_gregor_variant(row, gene_variants): return {'linked_variant': next( v['genetic_findings_id'] for v in gene_variants if v['genetic_findings_id'] != row['genetic_findings_id'] ) if len(gene_variants) > 1 else None} diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index a304da2407..c91bb4e3cf 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -611,7 +611,9 @@ ], [ 'Broad_HG00731_19_1912634', 'Broad_HG00731', 'Broad_exome_VCGS_FAM203_621_D2', 'SNV/INDEL', 'GRCh38', '19', '1912634', 'C', 'T', 'CA403171634', 'OR4G11P', 'ENST00000371839', '', '', 'Heterozygous', '', 'unknown', - 'Broad_HG00731_19_1912633', '', 'Known', '', 'MONDO:0044970', '', 'Full', '', '', 'SR-ES', '', '', '', '', '', '', '', + 'Broad_HG00731_19_1912633', '', 'Known', '', 'MONDO:0044970', '', 'Full', '', '', 'SR-ES', + 'The following variants are part of the multinucleotide variant 19-1912632-GC-TT (c.586_587delinsTT, p.Ala196Leu): 19-1912633-G-T, 19-1912634-C-T', + '', '', '', '', '', '', ], [ 'Broad_NA20889_1_248367227', 'Broad_NA20889', '', 'SNV/INDEL', 'GRCh37', '1', '248367227', 'TC', 'T', 'CA1501729', 'OR4G11P', 'ENST00000505820', 'c.3955G>A', 'c.1586-17C>G', 'Heterozygous', '', 'unknown', @@ -912,8 +914,9 @@ def test_gregor_export(self, mock_subprocess, mock_temp_dir, mock_open, mock_dat ], [ 'Broad_HG00731_19_1912634', 'Broad_HG00731', 'Broad_exome_VCGS_FAM203_621_D2', 'SNV/INDEL', 'GRCh38', '19', '1912634', 'C', 'T', 'CA403171634', 'OR4G11P', 'ENST00000371839', '', '', 'Heterozygous', '', 'unknown', - 'Broad_HG00731_19_1912633', '', 'Known', '', 'MONDO:0044970', '', 'Full', '', '', 'SR-ES', '', '', '', '', - '', '', '', + 'Broad_HG00731_19_1912633', '', 'Known', '', 'MONDO:0044970', '', 'Full', '', '', 'SR-ES', + 'The following variants are part of the multinucleotide variant 19-1912632-GC-TT (c.586_587delinsTT, p.Ala196Leu): 19-1912633-G-T, 19-1912634-C-T', + '', '', '', '', '', '', ]], additional_calls=2) responses.calls.reset() diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 77f0f3b4dc..d71bbb9cb0 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -305,23 +305,17 @@ def _get_genotype_zygosity(genotype): return None -def _post_process_variant_metadata(v, gene_variants, include_parent_mnvs=False): - discovery_notes = None - if len(gene_variants) > 2: - parent_mnv = next((v for v in gene_variants if len(v['individual_genotype']) == 1), gene_variants[0]) - if parent_mnv['genetic_findings_id'] == v['genetic_findings_id'] and not include_parent_mnvs: - return None - variant_type = 'complex structural' if parent_mnv.get('svType') else 'multinucleotide' - parent_name = _get_nested_variant_name(parent_mnv) - parent_details = [parent_mnv[key] for key in ['hgvsc', 'hgvsp'] if parent_mnv.get(key)] - parent = f'{parent_name} ({", ".join(parent_details)})' if parent_details else parent_name - mnv_names = [_get_nested_variant_name(v) for v in gene_variants] - nested_mnvs = sorted([v for v in mnv_names if v != parent_name]) - discovery_notes = f'The following variants are part of the {variant_type} variant {parent}: {", ".join(nested_mnvs)}' - return { - 'sv_name': _get_sv_name(v), - 'notes': discovery_notes, - } +def _get_discovery_notes(v, gene_variants, include_parent_mnvs): + parent_mnv = next((v for v in gene_variants if len(v['individual_genotype']) == 1), gene_variants[0]) + if parent_mnv['genetic_findings_id'] == v['genetic_findings_id'] and not include_parent_mnvs: + return None + variant_type = 'complex structural' if parent_mnv.get('svType') else 'multinucleotide' + parent_name = _get_nested_variant_name(parent_mnv) + parent_details = [parent_mnv[key] for key in ['hgvsc', 'hgvsp'] if parent_mnv.get(key)] + parent = f'{parent_name} ({", ".join(parent_details)})' if parent_details else parent_name + mnv_names = [_get_nested_variant_name(v) for v in gene_variants] + nested_mnvs = sorted([v for v in mnv_names if v != parent_name]) + return f'The following variants are part of the {variant_type} variant {parent}: {", ".join(nested_mnvs)}' def _get_parsed_saved_discovery_variants_by_family( @@ -376,6 +370,10 @@ def _get_parsed_saved_discovery_variants_by_family( parsed_variant.update({ 'seqr_chosen_consequence': main_transcript.get('majorConsequence'), }) + if include_svs: + parsed_variant.update({ + 'sv_name': _get_sv_name(parsed_variant), + }) variants.append(parsed_variant) genes_by_id = get_genes(gene_ids) @@ -510,12 +508,18 @@ def _get_genetic_findings_rows(rows: list[dict], individual: Individual, partici to_remove = [] for row in parsed_rows: del row['genotypes'] - process_func = post_process_variant or _post_process_variant_metadata - update = process_func(row, variants_by_gene[row[GENE_COLUMN]], include_parent_mnvs=include_parent_mnvs) - if update: - row.update(update) - else: - to_remove.append(row) + + gene_variants = variants_by_gene[row[GENE_COLUMN]] + discovery_notes = None + if len(gene_variants) > 2: + discovery_notes = _get_discovery_notes(row, gene_variants, include_parent_mnvs) + if discovery_notes is None: + to_remove.append(row) + continue + row['notes'] = discovery_notes + + if post_process_variant: + row.update(post_process_variant(row, gene_variants)) return [row for row in parsed_rows if row not in to_remove] From 2a00ffc93b9678adfae838b927a80975bbb2de14 Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Mon, 1 Jul 2024 14:00:39 -0400 Subject: [PATCH 51/67] use isinstance --- seqr/views/utils/terra_api_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/views/utils/terra_api_utils.py b/seqr/views/utils/terra_api_utils.py index 46d7ab80e7..314bbc4fec 100644 --- a/seqr/views/utils/terra_api_utils.py +++ b/seqr/views/utils/terra_api_utils.py @@ -108,7 +108,7 @@ def _safe_get_social(user): return None social = social_auth.first() - if type(social.extra_data) is str: # JSONField extra_data is returned as a string + if isinstance(social.extra_data, str): # JSONField extra_data is returned as a string social.extra_data = json.loads(social.extra_data) return social From 38a58dc82a5723f590359c3ace4c0530793c5287 Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Mon, 1 Jul 2024 14:01:59 -0400 Subject: [PATCH 52/67] Support only a single callset path (#4192) --- seqr/views/apis/anvil_workspace_api_tests.py | 4 ++-- seqr/views/apis/data_manager_api_tests.py | 4 +--- seqr/views/utils/airflow_utils.py | 2 +- seqr/views/utils/test_utils.py | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/seqr/views/apis/anvil_workspace_api_tests.py b/seqr/views/apis/anvil_workspace_api_tests.py index f24cc485d1..b29c40183f 100644 --- a/seqr/views/apis/anvil_workspace_api_tests.py +++ b/seqr/views/apis/anvil_workspace_api_tests.py @@ -741,7 +741,7 @@ def _assert_valid_operation(self, project, test_add_data=True): dag_json = { 'projects_to_run': [project.guid], - 'callset_paths': ['gs://test_bucket/test_path.vcf'], + 'callset_path': 'gs://test_bucket/test_path.vcf', 'sample_source': 'AnVIL', 'sample_type': 'WES', 'reference_genome': genome_version, @@ -824,7 +824,7 @@ def _test_mv_file_and_triggering_dag_exception(self, url, workspace, sample_data dag_id=self.DAG_NAME, dag=json.dumps({ 'projects_to_run': [project.guid], - 'callset_paths': ['gs://test_bucket/test_path.vcf'], + 'callset_path': 'gs://test_bucket/test_path.vcf', 'sample_source': 'AnVIL', 'sample_type': 'WES', 'reference_genome': genome_version, diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 881edc5dd7..f2d53ef5c3 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -1588,9 +1588,7 @@ def test_load_data(self, mock_subprocess, mock_temp_dir, mock_open): "R0001_1kg", "R0004_non_analyst_project" ], - "callset_paths": [ - "gs://test_bucket/mito_callset.mt" - ], + "callset_path": "gs://test_bucket/mito_callset.mt", "sample_source": "Broad_Internal", "sample_type": "WGS", "reference_genome": "GRCh38" diff --git a/seqr/views/utils/airflow_utils.py b/seqr/views/utils/airflow_utils.py index 1e7c0c11b3..7f27cbf4d5 100644 --- a/seqr/views/utils/airflow_utils.py +++ b/seqr/views/utils/airflow_utils.py @@ -34,7 +34,7 @@ def trigger_data_loading(projects: list[Project], sample_type: str, dataset_type project_guids = sorted([p.guid for p in projects]) updated_variables = { 'projects_to_run': project_guids, - 'callset_paths': [data_path], + 'callset_path': data_path, 'sample_source': 'Broad_Internal' if is_internal else 'AnVIL', 'sample_type': sample_type, 'reference_genome': GENOME_VERSION_LOOKUP[genome_version], diff --git a/seqr/views/utils/test_utils.py b/seqr/views/utils/test_utils.py index f0ed234d4f..93ab1f312d 100644 --- a/seqr/views/utils/test_utils.py +++ b/seqr/views/utils/test_utils.py @@ -659,7 +659,7 @@ def assert_airflow_calls(self, trigger_error=False, additional_tasks_check=False dag_variable_overrides = self._get_dag_variable_overrides(additional_tasks_check) dag_variables = { 'projects_to_run': [dag_variable_overrides['project']] if 'project' in dag_variable_overrides else self.PROJECTS, - 'callset_paths': [f'gs://test_bucket/{dag_variable_overrides["callset_path"]}'], + 'callset_path': f'gs://test_bucket/{dag_variable_overrides["callset_path"]}', 'sample_source': dag_variable_overrides['sample_source'], 'sample_type': dag_variable_overrides['sample_type'], 'reference_genome': dag_variable_overrides.get('reference_genome', 'GRCh38'), From 735c70398c081fd1165d5ac50bacbeca636a6ffd Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Mon, 1 Jul 2024 14:09:15 -0400 Subject: [PATCH 53/67] try setting --- seqr/views/utils/terra_api_utils.py | 10 ++-------- settings.py | 1 + 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/seqr/views/utils/terra_api_utils.py b/seqr/views/utils/terra_api_utils.py index 314bbc4fec..4a81f15c18 100644 --- a/seqr/views/utils/terra_api_utils.py +++ b/seqr/views/utils/terra_api_utils.py @@ -103,14 +103,8 @@ def _safe_get_social(user): if not google_auth_enabled() or not hasattr(user, 'social_auth'): return None - social_auth = user.social_auth.filter(provider=SOCIAL_AUTH_PROVIDER) - if not social_auth: - return None - - social = social_auth.first() - if isinstance(social.extra_data, str): # JSONField extra_data is returned as a string - social.extra_data = json.loads(social.extra_data) - return social + social = user.social_auth.filter(provider=SOCIAL_AUTH_PROVIDER) + return social.first() if social else None def _get_social_access_token(user): diff --git a/settings.py b/settings.py index a3e7e502f5..9c3a07e613 100644 --- a/settings.py +++ b/settings.py @@ -376,6 +376,7 @@ ######################################################### # Social auth specific settings ######################################################### +SOCIAL_AUTH_JSONFIELD_ENABLED = True SOCIAL_AUTH_GOOGLE_OAUTH2_IGNORE_DEFAULT_SCOPE = True SOCIAL_AUTH_GOOGLE_OAUTH2_SCOPE = [ 'https://www.googleapis.com/auth/userinfo.profile', From edaaf47a3b25be1f43be2b78210dd9b9dd178b46 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 1 Jul 2024 14:11:38 -0400 Subject: [PATCH 54/67] default include parent mnvs --- seqr/views/apis/report_api.py | 4 +--- seqr/views/utils/anvil_metadata_utils.py | 12 ++++++------ 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/seqr/views/apis/report_api.py b/seqr/views/apis/report_api.py index 12382aae14..fbc185b835 100644 --- a/seqr/views/apis/report_api.py +++ b/seqr/views/apis/report_api.py @@ -160,7 +160,7 @@ def _add_row(row, family_id, row_type): max_loaded_date = request.GET.get('loadedBefore') or (datetime.now() - timedelta(days=365)).strftime('%Y-%m-%d') parse_anvil_metadata( - [project], request.user, _add_row, max_loaded_date=max_loaded_date, include_discovery_sample_id=True, + [project], request.user, _add_row, max_loaded_date=max_loaded_date, include_discovery_sample_id=True, omit_parent_mnvs=True, get_additional_individual_fields=lambda individual, airtable_metadata, has_dbgap_submission, *args: { 'congenital_status': Individual.ONSET_AGE_LOOKUP[individual.onset_age] if individual.onset_age else 'Unknown', **anvil_export_airtable_fields(airtable_metadata, has_dbgap_submission), @@ -400,7 +400,6 @@ def _add_row(row, family_id, row_type): format_id=_format_gregor_id, get_additional_individual_fields=_get_participant_row, post_process_variant=_post_process_gregor_variant, - include_parent_mnvs=True, # TODO flag needed? include_svs=False, airtable_fields=[[PARTICIPANT_ID_FIELD, 'Recontactable'], [SMID_FIELD]], include_mondo=True, @@ -951,7 +950,6 @@ def _add_row(row, family_id, row_type): include_mondo=True, omit_airtable=True, proband_only_variants=True, - include_parent_mnvs=True, ) return create_json_response({'rows': variant_rows}) diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index d71bbb9cb0..2bb27b54f4 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -165,7 +165,7 @@ def parse_anvil_metadata( airtable_fields: Iterable[str] = None, mme_value: Aggregate = None, include_svs: bool = True, variant_json_fields: Iterable[str] = None, variant_attr_fields: Iterable[str] = None, post_process_variant: Callable[[dict, list[dict]], dict] = None, include_no_individual_families: bool = False, omit_airtable: bool = False, include_family_name_display: bool = False, include_family_sample_metadata: bool = False, - include_discovery_sample_id: bool = False, include_mondo: bool = False, include_parent_mnvs: bool = False, + include_discovery_sample_id: bool = False, include_mondo: bool = False, omit_parent_mnvs: bool = False, proband_only_variants: bool = False): individual_samples = individual_samples or (_get_loaded_before_date_project_individual_samples(projects, max_loaded_date) \ @@ -253,7 +253,7 @@ def parse_anvil_metadata( continue discovery_row = _get_genetic_findings_rows( saved_variants, individual, participant_id=participant_id, - format_id=format_id, include_parent_mnvs=include_parent_mnvs, + format_id=format_id, omit_parent_mnvs=omit_parent_mnvs, individual_data_types=(individual_data_types or {}).get(participant_id), family_individuals=family_individuals if proband_only_variants else None, sample=sample if include_discovery_sample_id else None, @@ -305,9 +305,9 @@ def _get_genotype_zygosity(genotype): return None -def _get_discovery_notes(v, gene_variants, include_parent_mnvs): +def _get_discovery_notes(variant, gene_variants, omit_parent_mnvs): parent_mnv = next((v for v in gene_variants if len(v['individual_genotype']) == 1), gene_variants[0]) - if parent_mnv['genetic_findings_id'] == v['genetic_findings_id'] and not include_parent_mnvs: + if parent_mnv['genetic_findings_id'] == variant['genetic_findings_id'] and omit_parent_mnvs: return None variant_type = 'complex structural' if parent_mnv.get('svType') else 'multinucleotide' parent_name = _get_nested_variant_name(parent_mnv) @@ -470,7 +470,7 @@ def _get_sample_row(sample, participant_id, has_dbgap_submission, airtable_metad def _get_genetic_findings_rows(rows: list[dict], individual: Individual, participant_id: str, individual_data_types: Iterable[str], family_individuals: dict[str, str], post_process_variant: Callable[[dict, list[dict]], dict], - format_id: Callable[[str], str], include_parent_mnvs: bool, sample: Sample) -> list[dict]: + format_id: Callable[[str], str], omit_parent_mnvs: bool, sample: Sample) -> list[dict]: parsed_rows = [] variants_by_gene = defaultdict(list) for row in (rows or []): @@ -512,7 +512,7 @@ def _get_genetic_findings_rows(rows: list[dict], individual: Individual, partici gene_variants = variants_by_gene[row[GENE_COLUMN]] discovery_notes = None if len(gene_variants) > 2: - discovery_notes = _get_discovery_notes(row, gene_variants, include_parent_mnvs) + discovery_notes = _get_discovery_notes(row, gene_variants, omit_parent_mnvs) if discovery_notes is None: to_remove.append(row) continue From 76379aa62c342c59c37ebac5547efc28bb21800d Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Mon, 1 Jul 2024 14:43:50 -0400 Subject: [PATCH 55/67] revert --- requirements.in | 2 +- requirements.txt | 2 +- seqr/views/utils/terra_api_utils.py | 10 ++++++++-- settings.py | 1 - 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/requirements.in b/requirements.in index 602875d970..3ba3fe97b2 100644 --- a/requirements.in +++ b/requirements.in @@ -5,7 +5,7 @@ django-guardian # object-level permissions for database record django-hijack # allows admins to login as other user django-notifications-hq # notification app django-cors-headers # allows CORS requests for client-side development -django-storages[google]==1.11.1 # alternative GCS storage backend for the django media_root +django-storages[google]==1.14.3 # alternative GCS storage backend for the django media_root social-auth-app-django # the package for Django to authenticate users with social medieas social-auth-core # the Python social authentication package. Required by social-auth-app-django elasticsearch==7.9.1 # elasticsearch client diff --git a/requirements.txt b/requirements.txt index f099f1b249..5066b63a47 100644 --- a/requirements.txt +++ b/requirements.txt @@ -52,7 +52,7 @@ django-model-utils==4.3.1 # via django-notifications-hq django-notifications-hq==1.8.3 # via -r requirements.in -django-storages[google]==1.11.1 +django-storages[google]==1.14.3 # via -r requirements.in elasticsearch==7.9.1 # via diff --git a/seqr/views/utils/terra_api_utils.py b/seqr/views/utils/terra_api_utils.py index 4a81f15c18..314bbc4fec 100644 --- a/seqr/views/utils/terra_api_utils.py +++ b/seqr/views/utils/terra_api_utils.py @@ -103,8 +103,14 @@ def _safe_get_social(user): if not google_auth_enabled() or not hasattr(user, 'social_auth'): return None - social = user.social_auth.filter(provider=SOCIAL_AUTH_PROVIDER) - return social.first() if social else None + social_auth = user.social_auth.filter(provider=SOCIAL_AUTH_PROVIDER) + if not social_auth: + return None + + social = social_auth.first() + if isinstance(social.extra_data, str): # JSONField extra_data is returned as a string + social.extra_data = json.loads(social.extra_data) + return social def _get_social_access_token(user): diff --git a/settings.py b/settings.py index 9c3a07e613..a3e7e502f5 100644 --- a/settings.py +++ b/settings.py @@ -376,7 +376,6 @@ ######################################################### # Social auth specific settings ######################################################### -SOCIAL_AUTH_JSONFIELD_ENABLED = True SOCIAL_AUTH_GOOGLE_OAUTH2_IGNORE_DEFAULT_SCOPE = True SOCIAL_AUTH_GOOGLE_OAUTH2_SCOPE = [ 'https://www.googleapis.com/auth/userinfo.profile', From bf6efba817f0edc193f08afef1fa9b5ab28008c1 Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Mon, 1 Jul 2024 14:53:17 -0400 Subject: [PATCH 56/67] try pinning sqlparse --- requirements.in | 1 + requirements.txt | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/requirements.in b/requirements.in index 3ba3fe97b2..5fda7b8d07 100644 --- a/requirements.in +++ b/requirements.in @@ -8,6 +8,7 @@ django-cors-headers # allows CORS requests for client-side develop django-storages[google]==1.14.3 # alternative GCS storage backend for the django media_root social-auth-app-django # the package for Django to authenticate users with social medieas social-auth-core # the Python social authentication package. Required by social-auth-app-django +sqlparse==0.5.0 # required by Django, pinned to version without vulnerabilities elasticsearch==7.9.1 # elasticsearch client elasticsearch-dsl==7.2.1 # elasticsearch query utilities gunicorn # web server diff --git a/requirements.txt b/requirements.txt index 5066b63a47..1c3db4627c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -166,7 +166,9 @@ social-auth-core==4.3.0 soupsieve==2.5 # via beautifulsoup4 sqlparse==0.5.0 - # via django + # via + # -r requirements.in + # django swapper==1.3.0 # via django-notifications-hq tenacity==8.3.0 From ae5fa255ecfc91c4abe017d56664b710f35a9b8e Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Mon, 1 Jul 2024 14:54:52 -0400 Subject: [PATCH 57/67] revert --- requirements.in | 1 - requirements.txt | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/requirements.in b/requirements.in index 5fda7b8d07..3ba3fe97b2 100644 --- a/requirements.in +++ b/requirements.in @@ -8,7 +8,6 @@ django-cors-headers # allows CORS requests for client-side develop django-storages[google]==1.14.3 # alternative GCS storage backend for the django media_root social-auth-app-django # the package for Django to authenticate users with social medieas social-auth-core # the Python social authentication package. Required by social-auth-app-django -sqlparse==0.5.0 # required by Django, pinned to version without vulnerabilities elasticsearch==7.9.1 # elasticsearch client elasticsearch-dsl==7.2.1 # elasticsearch query utilities gunicorn # web server diff --git a/requirements.txt b/requirements.txt index 1c3db4627c..5066b63a47 100644 --- a/requirements.txt +++ b/requirements.txt @@ -166,9 +166,7 @@ social-auth-core==4.3.0 soupsieve==2.5 # via beautifulsoup4 sqlparse==0.5.0 - # via - # -r requirements.in - # django + # via django swapper==1.3.0 # via django-notifications-hq tenacity==8.3.0 From 0c6df993109ad9f38fef5a680c8a52eb6fc16517 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 1 Jul 2024 15:47:04 -0400 Subject: [PATCH 58/67] add publication summary to discovery notes --- seqr/views/apis/report_api_tests.py | 39 ++++++++++++++++++++--- seqr/views/apis/summary_data_api_tests.py | 6 ++-- seqr/views/utils/anvil_metadata_utils.py | 12 ++++--- 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index c91bb4e3cf..5b074c0be9 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -504,7 +504,7 @@ 'hgvsc': '', 'hgvsp': '', 'method_of_discovery': 'SR-ES', - 'notes': None, + 'notes': '', 'phenotype_contribution': 'Full', 'partial_contribution_explained': '', 'seqr_chosen_consequence': None, @@ -603,7 +603,7 @@ 'Broad_NA19675_1_21_3343353', 'Broad_NA19675_1', '', 'SNV/INDEL', 'GRCh37', '21', '3343353', 'GAGA', 'G', '', 'RP11', 'ENST00000258436.5', 'c.375_377delTCT', 'p.Leu126del', 'Heterozygous', '', 'de novo', '', '', 'Candidate', 'Myasthenic syndrome, congenital, 8, with pre- and postsynaptic defects', 'OMIM:615120', 'Autosomal recessive|X-linked', - 'Full', '', '', 'SR-ES', '', '', '', '', '', '', '', + 'Full', '', '', 'SR-ES', 'This individual is published in PMID34415322', '', '', '', '', '', '', ], [ 'Broad_HG00731_1_248367227', 'Broad_HG00731', 'Broad_exome_VCGS_FAM203_621_D2', 'SNV/INDEL', 'GRCh37', '1', '248367227', 'TC', 'T', 'CA1501729', 'RP11', '', '', '', 'Homozygous', '', 'paternal', '', '', 'Known', '', @@ -746,7 +746,7 @@ def test_anvil_export(self, mock_google_authenticated, mock_zip): self.assertIn([ '21_3343353_NA19675_1', 'NA19675_1', 'NA19675', 'RP11', 'Candidate', 'de novo', 'Heterozygous', 'GRCh37', '21', '3343353', 'GAGA', 'G', 'c.375_377delTCT', 'p.Leu126del', 'ENST00000258436.5', - '-', '-', '-', '-'], discovery_file) + '-', '-', '-', 'This individual is published in PMID34415322'], discovery_file) self.assertIn([ '19_1912633_HG00731', 'HG00731', 'HG00731', 'OR4G11P', 'Known', 'unknown', 'Heterozygous', 'GRCh38', '19', '1912633', 'G', 'T', '-', '-', 'ENST00000371839', '-', '-', '-', @@ -906,7 +906,8 @@ def test_gregor_export(self, mock_subprocess, mock_temp_dir, mock_open, mock_dat 'Broad_NA19675_1_21_3343353', 'Broad_NA19675_1', '', 'SNV/INDEL', 'GRCh37', '21', '3343353', 'GAGA', 'G', '', 'RP11', 'ENST00000258436.5', 'c.375_377delTCT', 'p.Leu126del', 'Heterozygous', '', 'de novo', '', '', 'Candidate', 'Myasthenic syndrome, congenital, 8, with pre- and postsynaptic defects', 'OMIM:615120', - 'Autosomal recessive|X-linked', 'Full', '', '', 'SR-ES', '', '', '', '', '', '', '', + 'Autosomal recessive|X-linked', 'Full', '', '', 'SR-ES', 'This individual is published in PMID34415322', + '', '', '', '', '', '', ], [ 'Broad_HG00731_1_248367227', 'Broad_HG00731', 'Broad_exome_VCGS_FAM203_621_D2', 'SNV/INDEL', 'GRCh37', '1', '248367227', 'TC', 'T', 'CA1501729', 'RP11', '', '', '', 'Homozygous', '', 'paternal', '', '', 'Known', '', @@ -1259,6 +1260,36 @@ def test_variant_metadata(self): self.assertListEqual(list(response_json.keys()), ['rows']) row_ids = ['NA19675_1_21_3343353', 'HG00731_1_248367227', 'HG00731_19_1912634', 'HG00731_19_1912633', 'HG00731_19_1912632'] self.assertListEqual([r['genetic_findings_id'] for r in response_json['rows']], row_ids) + self.assertDictEqual(response_json['rows'][0], { + **BASE_VARIANT_METADATA_ROW, + 'alt': 'G', + 'chrom': '21', + 'clinvar': {'alleleId': None, 'clinicalSignificance': '', 'goldStars': None, 'variationId': None}, + 'condition_id': 'OMIM:615120', + 'condition_inheritance': 'Autosomal recessive|X-linked', + 'displayName': '1', + 'familyGuid': 'F000001_1', + 'family_id': '1', + 'gene_of_interest': 'RP11', + 'gene_id': 'ENSG00000135953', + 'gene_known_for_phenotype': 'Candidate', + 'genetic_findings_id': 'NA19675_1_21_3343353', + 'hgvsc': 'c.375_377delTCT', + 'hgvsp': 'p.Leu126del', + 'known_condition_name': 'Myasthenic syndrome, congenital, 8, with pre- and postsynaptic defects', + 'MME': True, + 'notes': 'This individual is published in PMID34415322', + 'participant_id': 'NA19675_1', + 'pos': 3343353, + 'projectGuid': 'R0001_1kg', + 'ref': 'GAGA', + 'seqr_chosen_consequence': 'inframe_deletion', + 'tags': ['Tier 1 - Novel gene and phenotype'], + 'transcript': 'ENST00000258436.5', + 'variant_inheritance': 'de novo', + 'variant_reference_assembly': 'GRCh37', + 'zygosity': 'Heterozygous', + }) expected_row = { **BASE_VARIANT_METADATA_ROW, 'additional_family_members_with_variant': 'HG00732', diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index 835c9ac946..350e218dd6 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -98,8 +98,8 @@ 'svType-1': None, 'sv_name-1': None, 'end-1': None, - 'notes-1': None, - 'notes-2': None, + 'notes-1': '', + 'notes-2': '', 'phenotype_contribution-1': 'Partial', 'phenotype_contribution-2': 'Full', 'partial_contribution_explained-1': 'HP:0000501|HP:0000365', @@ -161,7 +161,7 @@ 'gene_id-1': None, 'hgvsc-1': '', 'hgvsp-1': '', - 'notes-1': None, + 'notes-1': '', 'seqr_chosen_consequence-1': None, 'svType-1': None, 'sv_name-1': None, diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 2bb27b54f4..2f47aca5d0 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -252,7 +252,7 @@ def parse_anvil_metadata( if proband_only_variants and individual.proband_relationship != Individual.SELF_RELATIONSHIP: continue discovery_row = _get_genetic_findings_rows( - saved_variants, individual, participant_id=participant_id, + saved_variants, individual, subject_family_row, participant_id=participant_id, format_id=format_id, omit_parent_mnvs=omit_parent_mnvs, individual_data_types=(individual_data_types or {}).get(participant_id), family_individuals=family_individuals if proband_only_variants else None, @@ -467,7 +467,7 @@ def _get_sample_row(sample, participant_id, has_dbgap_submission, airtable_metad return sample_row -def _get_genetic_findings_rows(rows: list[dict], individual: Individual, participant_id: str, +def _get_genetic_findings_rows(rows: list[dict], individual: Individual, family_row: dict, participant_id: str, individual_data_types: Iterable[str], family_individuals: dict[str, str], post_process_variant: Callable[[dict, list[dict]], dict], format_id: Callable[[str], str], omit_parent_mnvs: bool, sample: Sample) -> list[dict]: @@ -510,13 +510,17 @@ def _get_genetic_findings_rows(rows: list[dict], individual: Individual, partici del row['genotypes'] gene_variants = variants_by_gene[row[GENE_COLUMN]] - discovery_notes = None + notes = [] if len(gene_variants) > 2: discovery_notes = _get_discovery_notes(row, gene_variants, omit_parent_mnvs) if discovery_notes is None: to_remove.append(row) continue - row['notes'] = discovery_notes + else: + notes.append(discovery_notes) + if family_row['pmid_id']: + notes.append(f'This individual is published in PMID{family_row["pmid_id"]}') + row['notes'] = '. '.join(notes) if post_process_variant: row.update(post_process_variant(row, gene_variants)) From d6546949db1dc342360e8fd7a747ca2da60ff401 Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Tue, 2 Jul 2024 12:25:43 -0400 Subject: [PATCH 59/67] add seqr notification to rna data loading --- seqr/views/apis/data_manager_api_tests.py | 39 ++++++++++++++--------- seqr/views/utils/dataset_utils.py | 21 +++++++++--- 2 files changed, 41 insertions(+), 19 deletions(-) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index f2d53ef5c3..34ffec4b53 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -16,7 +16,6 @@ from seqr.models import Individual, RnaSeqOutlier, RnaSeqTpm, RnaSeqSpliceOutlier, Sample, Project, PhenotypePrioritization from settings import SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL -SEQR_URL = 'https://seqr.broadinstitute.org/' PROJECT_GUID = 'R0001_1kg' ES_CAT_ALLOCATION=[{ @@ -873,6 +872,7 @@ def test_update_rna_splice_outlier(self, *args, **kwargs): @mock.patch('seqr.views.utils.dataset_utils.BASE_URL', 'https://test-seqr.org/') @mock.patch('seqr.views.utils.dataset_utils.SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL', 'seqr-data-loading') @mock.patch('seqr.views.utils.file_utils.tempfile.gettempdir', lambda: 'tmp/') + @mock.patch('seqr.utils.communication_utils.send_html_email') @mock.patch('seqr.views.utils.dataset_utils.safe_post_to_slack') @mock.patch('seqr.views.apis.data_manager_api.datetime') @mock.patch('seqr.views.apis.data_manager_api.os.mkdir') @@ -881,7 +881,7 @@ def test_update_rna_splice_outlier(self, *args, **kwargs): @mock.patch('seqr.utils.file_utils.subprocess.Popen') @mock.patch('seqr.views.apis.data_manager_api.gzip.open') def _test_update_rna_seq(self, data_type, mock_open, mock_subprocess, mock_load_uploaded_file, - mock_rename, mock_mkdir, mock_datetime, mock_send_slack): + mock_rename, mock_mkdir, mock_datetime, mock_send_slack, mock_send_email): url = reverse(update_rna_seq) self.check_pm_login(url) @@ -966,6 +966,7 @@ def _set_file_iter_stdout(rows): self._has_expected_file_loading_logs('gs://rna_data/muscle_samples.tsv.gz', info=info, warnings=warnings, user=self.pm_user) self.assertEqual(model_cls.objects.count(), params['initial_model_count']) mock_send_slack.assert_not_called() + mock_send_email.assert_not_called() self.assertEqual(mock_subprocess.call_count, 2) mock_subprocess.assert_has_calls([mock.call(command, stdout=-1, stderr=-2, shell=True) for command in [ # nosec f'gsutil ls {body["file"]}', @@ -1031,13 +1032,21 @@ def _test_basic_data_loading(data, num_parsed_samples, num_loaded_samples, new_s mock_send_slack.assert_has_calls([ mock.call( 'seqr-data-loading', - f'0 new RNA {params["message_data_type"]} samples are loaded in \n``````', + f'0 new RNA {params["message_data_type"]} samples are loaded in 1kg project nåme with uniçøde\n``````', ), mock.call( 'seqr-data-loading', - f'1 new RNA {params["message_data_type"]} samples are loaded in \n```NA20888```', + f'1 new RNA {params["message_data_type"]} samples are loaded in Test Reprocessed Project\n```NA20888```', ), ]) + self.assertEqual(mock_send_email.call_count, 2) + self._assert_expected_notifications(mock_send_email, [ + {'data_type': f'RNA {params["message_data_type"]}', 'user': self.data_manager_user, + 'email_body': f'data for 0 new RNA {params["message_data_type"]} sample(s)'}, + {'data_type': f'RNA {params["message_data_type"]}', 'user': self.data_manager_user, + 'email_body': f'data for 1 new RNA {params["message_data_type"]} sample(s)', + 'project_guid': 'R0003_test', 'project_name': 'Test Reprocessed Project'} + ]) # test database models are correct self.assertEqual(model_cls.objects.count(), params['initial_model_count'] - deleted_count) @@ -1187,7 +1196,7 @@ def test_load_rna_seq_sample_data(self): def _join_data(cls, data): return ['\t'.join(line).encode('utf-8') for line in data] - @mock.patch('seqr.views.apis.data_manager_api.BASE_URL', SEQR_URL) + @mock.patch('seqr.views.apis.data_manager_api.BASE_URL', 'https://test-seqr.org/') @mock.patch('seqr.models.random') @mock.patch('seqr.utils.communication_utils.send_html_email') @mock.patch('seqr.utils.file_utils.subprocess.Popen') @@ -1270,8 +1279,8 @@ def test_load_phenotype_prioritization_data(self, mock_subprocess, mock_send_ema self.assertListEqual(saved_data, EXPECTED_LIRICAL_DATA) mock_subprocess.assert_called_with('gsutil cat gs://seqr_data/lirical_data.tsv.gz | gunzip -c -q - ', stdout=-1, stderr=-2, shell=True) # nosec self._assert_expected_notifications(mock_send_email, [ - {'tool': 'lirical', 'num_samples': 1, 'user': self.data_manager_user}, - {'tool': 'lirical', 'num_samples': 1, 'user': self.data_manager_user, + {'data_type': 'Lirical', 'user': self.data_manager_user, 'email_body': 'Lirical data for 1 sample(s)'}, + {'data_type': 'Lirical', 'user': self.data_manager_user, 'email_body': 'Lirical data for 1 sample(s)', 'project_guid': 'R0003_test', 'project_name': 'Test Reprocessed Project'} ]) @@ -1301,7 +1310,7 @@ def test_load_phenotype_prioritization_data(self, mock_subprocess, mock_send_ema nested_fields=[{'fields': ('individual', 'guid'), 'key': 'individualGuid'}]) self.assertListEqual(saved_data, EXPECTED_UPDATED_LIRICAL_DATA) self._assert_expected_notifications(mock_send_email, [ - {'tool': 'lirical', 'num_samples': 2, 'user': self.data_manager_user}, + {'data_type': 'Lirical', 'user': self.data_manager_user, 'email_body': 'Lirical data for 2 sample(s)'}, ]) @staticmethod @@ -1310,16 +1319,16 @@ def _assert_expected_notifications(mock_send_email, expected_notifs: list[dict]) for notif_dict in expected_notifs: project_guid = notif_dict.get('project_guid', PROJECT_GUID) project_name = notif_dict.get('project_name', '1kg project nåme with uniçøde') - url = f'{SEQR_URL}project/{project_guid}/project_page' + url = f'https://test-seqr.org/project/{project_guid}/project_page' project_link = f'{project_name}' - email = ( - f'This is to notify you that {notif_dict["tool"].title()} data for {notif_dict["num_samples"]} sample(s) ' - f'has been loaded in seqr project {project_link}' + expected_email_body = ( + f'Dear seqr user,\n\nThis is to notify you that {notif_dict["email_body"]} ' + f'has been loaded in seqr project {project_link}\n\nAll the best,\nThe seqr team' ) calls.append( mock.call( - email_body=f'Dear seqr user,\n\n{email}\n\nAll the best,\nThe seqr team', - subject=f'New {notif_dict["tool"].title()} data available in seqr', + email_body=expected_email_body, + subject=f'New {notif_dict["data_type"]} data available in seqr', to=['test_user_manager@test.com'], process_message=_set_bulk_notification_stream, ) diff --git a/seqr/views/utils/dataset_utils.py b/seqr/views/utils/dataset_utils.py index 4fa33c2048..f49eb1f086 100644 --- a/seqr/views/utils/dataset_utils.py +++ b/seqr/views/utils/dataset_utils.py @@ -5,7 +5,7 @@ from tqdm import tqdm from seqr.models import Sample, Individual, Family, Project, RnaSeqOutlier, RnaSeqTpm, RnaSeqSpliceOutlier -from seqr.utils.communication_utils import safe_post_to_slack +from seqr.utils.communication_utils import safe_post_to_slack, send_project_notification from seqr.utils.file_utils import file_iter from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException @@ -519,7 +519,7 @@ def match_sample(sample_key, unmatched_samples, sample_id_to_individual_id_mappi info.append(message) logger.info(message, user) - _notify_rna_loading(model_cls, sample_projects) + _notify_rna_loading(model_cls, sample_projects, projects) for warning in warnings: logger.warning(warning, user) @@ -564,15 +564,28 @@ def post_process_rna_data(sample_guid, data, get_unique_key=None, format_fields= RnaSeqTpm: 'Expression', } -def _notify_rna_loading(model_cls, sample_projects): + +def _notify_rna_loading(model_cls, sample_projects, internal_projects): + projects_by_name = {project.name: project for project in internal_projects} data_type = RNA_MODEL_DISPLAY_NAME[model_cls] for project_agg in sample_projects: new_ids = project_agg["new_sample_ids"] - project_link = f'<{BASE_URL}project/{project_agg["guid"]}/project_page|{project_agg["name"]}>' + url = f'{BASE_URL}project/{project_agg["guid"]}/project_page' + project_link = f'{project_agg["name"]}' safe_post_to_slack( SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL, f'{len(new_ids)} new RNA {data_type} samples are loaded in {project_link}\n```{", ".join(new_ids)}```' ) + email = ( + f'This is to notify you that data for {len(new_ids)} new RNA {data_type} sample(s) ' + f'has been loaded in seqr project {project_link}' + ) + send_project_notification( + project=projects_by_name[project_agg["name"]], + notification=f'Loaded {len(new_ids)} new RNA {data_type} sample(s)', + email=email, + subject=f'New RNA {data_type} data available in seqr', + ) PHENOTYPE_PRIORITIZATION_HEADER = ['tool', 'project', 'sampleId', 'rank', 'geneId', 'diseaseId', 'diseaseName'] From 32245d984d03a246c37684969b3e2e4dd153b395 Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Tue, 2 Jul 2024 12:46:50 -0400 Subject: [PATCH 60/67] be more lax with requirements --- requirements-dev.in | 10 +++++----- requirements-dev.txt | 2 +- requirements.in | 4 ++-- requirements.txt | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/requirements-dev.in b/requirements-dev.in index 7d689c6013..f85a139ed5 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -1,8 +1,8 @@ -c requirements.txt # use the generated reqs as a constraint coverage<5.2 django-compressor -django-debug-toolbar==4.4.2 # https://github.com/jazzband/django-debug-toolbar -mock # mock objects for unit tests -pip-tools # tool for managing our python dependency tree -responses # mock HTTP responses for unit tests -urllib3-mock # mock urllib3 for tests +django-debug-toolbar # https://github.com/jazzband/django-debug-toolbar +mock # mock objects for unit tests +pip-tools # tool for managing our python dependency tree +responses # mock HTTP responses for unit tests +urllib3-mock # mock urllib3 for tests diff --git a/requirements-dev.txt b/requirements-dev.txt index a1399355e1..5ef5890f4a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -31,7 +31,7 @@ django-appconf==1.0.5 # via django-compressor django-compressor==4.3.1 # via -r requirements-dev.in -django-debug-toolbar==4.4.2 +django-debug-toolbar==3.2.4 # via -r requirements-dev.in idna==3.7 # via diff --git a/requirements.in b/requirements.in index 3ba3fe97b2..e38886e965 100644 --- a/requirements.in +++ b/requirements.in @@ -1,11 +1,11 @@ -Django==4.2.13 # core server-side framework +Django>=4.2,<4.3 # core server-side framework django-anymail # for sending emails using cloud-based mail service providers django-csp # for setting CSP headers django-guardian # object-level permissions for database records. Behind a major version due to missing Python 2 support django-hijack # allows admins to login as other user django-notifications-hq # notification app django-cors-headers # allows CORS requests for client-side development -django-storages[google]==1.14.3 # alternative GCS storage backend for the django media_root +django-storages[google] # alternative GCS storage backend for the django media_root social-auth-app-django # the package for Django to authenticate users with social medieas social-auth-core # the Python social authentication package. Required by social-auth-app-django elasticsearch==7.9.1 # elasticsearch client diff --git a/requirements.txt b/requirements.txt index 5066b63a47..f099f1b249 100644 --- a/requirements.txt +++ b/requirements.txt @@ -52,7 +52,7 @@ django-model-utils==4.3.1 # via django-notifications-hq django-notifications-hq==1.8.3 # via -r requirements.in -django-storages[google]==1.14.3 +django-storages[google]==1.11.1 # via -r requirements.in elasticsearch==7.9.1 # via From b4777d4bc9cd575e00134ae8d338e200a90a8230 Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Tue, 2 Jul 2024 13:57:36 -0400 Subject: [PATCH 61/67] be explicit abuot storages --- settings.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/settings.py b/settings.py index bec2499f88..0679ec0cb6 100644 --- a/settings.py +++ b/settings.py @@ -136,16 +136,16 @@ 'django.contrib.staticfiles.finders.FileSystemFinder', 'django.contrib.staticfiles.finders.AppDirectoriesFinder', ) +STORAGES = { + 'default': {'BACKEND': 'django.core.files.storage.FileSystemStorage'}, + 'staticfiles': {'BACKEND': 'django.contrib.staticfiles.storage.StaticFilesStorage'} +} # If specified, store data in the named GCS bucket and use the gcloud storage backend. # Else, fall back to a path on the local filesystem. GCS_MEDIA_ROOT_BUCKET = os.environ.get('GCS_MEDIA_ROOT_BUCKET') if GCS_MEDIA_ROOT_BUCKET: - STORAGES = { - "default": { - "BACKEND": "storages.backends.gcloud.GoogleCloudStorage", - } - } + STORAGES['default'] = {'BACKEND': 'storages.backends.gcloud.GoogleCloudStorage'} GS_BUCKET_NAME = GCS_MEDIA_ROOT_BUCKET GS_DEFAULT_ACL = 'publicRead' MEDIA_ROOT = False From 813a379878950d0f4ac0f6dae845a68033e366ae Mon Sep 17 00:00:00 2001 From: Julia Klugherz Date: Tue, 2 Jul 2024 14:34:26 -0400 Subject: [PATCH 62/67] bump social auth and fix fixtures --- requirements.in | 2 +- requirements.txt | 7 ++--- seqr/fixtures/social_auth.json | 42 ++++++++++++++++++++++++----- seqr/views/utils/terra_api_utils.py | 10 +------ settings.py | 1 + 5 files changed, 43 insertions(+), 19 deletions(-) diff --git a/requirements.in b/requirements.in index e38886e965..b40fcdafc4 100644 --- a/requirements.in +++ b/requirements.in @@ -6,7 +6,7 @@ django-hijack # allows admins to login as other user django-notifications-hq # notification app django-cors-headers # allows CORS requests for client-side development django-storages[google] # alternative GCS storage backend for the django media_root -social-auth-app-django # the package for Django to authenticate users with social medieas +social-auth-app-django>5.0.0 # the package for Django to authenticate users with social medieas social-auth-core # the Python social authentication package. Required by social-auth-app-django elasticsearch==7.9.1 # elasticsearch client elasticsearch-dsl==7.2.1 # elasticsearch query utilities diff --git a/requirements.txt b/requirements.txt index f099f1b249..4abc425ee9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,6 +38,7 @@ django==4.2.13 # django-notifications-hq # django-storages # jsonfield + # social-auth-app-django django-anymail==9.0 # via -r requirements.in django-cors-headers==3.13.0 @@ -117,7 +118,7 @@ pyasn1-modules==0.2.8 # via google-auth pycparser==2.21 # via cffi -pyjwt==2.6.0 +pyjwt==2.8.0 # via social-auth-core pyliftover==0.4 # via -r requirements.in @@ -157,9 +158,9 @@ slacker==0.14.0 # via -r requirements.in slugify==0.0.1 # via -r requirements.in -social-auth-app-django==5.0.0 +social-auth-app-django==5.4.1 # via -r requirements.in -social-auth-core==4.3.0 +social-auth-core==4.5.4 # via # -r requirements.in # social-auth-app-django diff --git a/seqr/fixtures/social_auth.json b/seqr/fixtures/social_auth.json index 7d951c13ef..e092f257fa 100644 --- a/seqr/fixtures/social_auth.json +++ b/seqr/fixtures/social_auth.json @@ -6,7 +6,12 @@ "user": 10, "provider": "google-oauth2", "uid": "test_user@broadinstitute.org", - "extra_data": "{\"expires\": 3599, \"auth_time\": 1603287741, \"token_type\": \"Bearer\", \"access_token\": \"ya29.EXAMPLE\"}", + "extra_data": { + "expires": 3599, + "auth_time": 1603287741, + "token_type": "Bearer", + "access_token": "ya29.EXAMPLE" + }, "created": "2020-03-12T23:09:54.180Z", "modified": "2020-03-12T23:09:54.180Z" } @@ -17,7 +22,12 @@ "user": 11, "provider": "google-oauth2", "uid": "test_user_manager@test.com", - "extra_data": "{\"expires\": 3599, \"auth_time\": 1603287741, \"token_type\": \"Bearer\", \"access_token\": \"ya29.EXAMPLE\"}", + "extra_data": { + "expires": 3599, + "auth_time": 1603287741, + "token_type": "Bearer", + "access_token": "ya29.EXAMPLE" + }, "created": "2020-03-12T23:09:54.180Z", "modified": "2020-03-12T23:09:54.180Z" } @@ -28,7 +38,12 @@ "user": 12, "provider": "google-oauth2", "uid": "test_user_no_staff@test.com", - "extra_data": "{\"expires\": 3599, \"auth_time\": 1603287741, \"token_type\": \"Bearer\", \"access_token\": \"ya29.EXAMPLE\"}", + "extra_data": { + "expires": 6666, + "auth_time": 1603287741, + "token_type": "Bearer", + "access_token": "ya29.EXAMPLE" + }, "created": "2020-03-12T23:09:54.180Z", "modified": "2020-03-12T23:09:54.180Z" } @@ -39,7 +54,12 @@ "user": 13, "provider": "google-oauth2", "uid": "test_user_no_access@test.com", - "extra_data": "{\"expires\": 3599, \"auth_time\": 1603287741, \"token_type\": \"Bearer\", \"access_token\": \"ya29.EXAMPLE\"}", + "extra_data": { + "expires": 3599, + "auth_time": 1603287741, + "token_type": "Bearer", + "access_token": "ya29.EXAMPLE" + }, "created": "2020-03-12T23:09:54.180Z", "modified": "2020-03-12T23:09:54.180Z" } @@ -50,7 +70,12 @@ "user": 17, "provider": "google-oauth2", "uid": "test_pm_user@test.com", - "extra_data": "{\"expires\": 3599, \"auth_time\": 1603287741, \"token_type\": \"Bearer\", \"access_token\": \"ya29.EXAMPLE\"}", + "extra_data": { + "expires": 3599, + "auth_time": 1603287741, + "token_type": "Bearer", + "access_token": "ya29.EXAMPLE" + }, "created": "2020-03-12T23:09:54.180Z", "modified": "2020-03-12T23:09:54.180Z" } @@ -61,7 +86,12 @@ "user": 15, "provider": "google-oauth2", "uid": "test_superuser@test.com", - "extra_data": "{\"expires\": 3599, \"auth_time\": 1603287741, \"token_type\": \"Bearer\", \"access_token\": \"ya29.EXAMPLE\"}", + "extra_data": { + "expires": 3599, + "auth_time": 1603287741, + "token_type": "Bearer", + "access_token": "ya29.EXAMPLE" + }, "created": "2020-03-12T23:09:54.180Z", "modified": "2020-03-12T23:09:54.180Z" } diff --git a/seqr/views/utils/terra_api_utils.py b/seqr/views/utils/terra_api_utils.py index 314bbc4fec..9ec6427254 100644 --- a/seqr/views/utils/terra_api_utils.py +++ b/seqr/views/utils/terra_api_utils.py @@ -102,15 +102,7 @@ def _get_call_args(path, headers=None, root_url=None): def _safe_get_social(user): if not google_auth_enabled() or not hasattr(user, 'social_auth'): return None - - social_auth = user.social_auth.filter(provider=SOCIAL_AUTH_PROVIDER) - if not social_auth: - return None - - social = social_auth.first() - if isinstance(social.extra_data, str): # JSONField extra_data is returned as a string - social.extra_data = json.loads(social.extra_data) - return social + return user.social_auth.filter(provider=SOCIAL_AUTH_PROVIDER).first() def _get_social_access_token(user): diff --git a/settings.py b/settings.py index 0679ec0cb6..a0b1fc001d 100644 --- a/settings.py +++ b/settings.py @@ -378,6 +378,7 @@ ######################################################### # Social auth specific settings ######################################################### +SOCIAL_AUTH_JSONFIELD_ENABLED = True SOCIAL_AUTH_GOOGLE_OAUTH2_IGNORE_DEFAULT_SCOPE = True SOCIAL_AUTH_GOOGLE_OAUTH2_SCOPE = [ 'https://www.googleapis.com/auth/userinfo.profile', From 27698f536b5c80e75b674cc06488d91beff95b67 Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Wed, 3 Jul 2024 11:39:38 -0400 Subject: [PATCH 63/67] lint docker compose (#4213) --- .github/workflows/docker-lint.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-lint.yaml b/.github/workflows/docker-lint.yaml index ab1399ba69..a0b1e5de9f 100644 --- a/.github/workflows/docker-lint.yaml +++ b/.github/workflows/docker-lint.yaml @@ -11,6 +11,7 @@ on: - deploy/docker/seqr/Dockerfile - hail_search/deploy/Dockerfile - .hadolint.yaml + - .docker-compose.yaml - .github/workflows/docker-lint.yaml pull_request: types: [opened, synchronize, reopened] @@ -21,13 +22,16 @@ on: - deploy/docker/seqr/Dockerfile - hail_search/deploy/Dockerfile - .hadolint.yaml + - .docker-compose.yaml - .github/workflows/docker-lint.yaml jobs: hadolint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v2 + - name: Validate docker compose + run: docker-compose -f docker-compose.yml config - uses: hadolint/hadolint-action@v1.5.0 with: dockerfile: deploy/docker/seqr/Dockerfile From d55aa3b859f7916716f94a97868f9a76b8d7d7b0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 3 Jul 2024 14:44:54 -0400 Subject: [PATCH 64/67] update display for extended splice region --- .../components/panel/variants/Annotations.jsx | 10 ++++++++-- .../components/panel/variants/Transcripts.jsx | 14 ++++---------- ui/shared/utils/constants.js | 4 +++- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/ui/shared/components/panel/variants/Annotations.jsx b/ui/shared/components/panel/variants/Annotations.jsx index 5581233aca..8ce120aca1 100644 --- a/ui/shared/components/panel/variants/Annotations.jsx +++ b/ui/shared/components/panel/variants/Annotations.jsx @@ -22,7 +22,7 @@ import Modal from '../../modal/Modal' import { ButtonLink, HelpIcon } from '../../StyledComponents' import RnaSeqJunctionOutliersTable from '../../table/RnaSeqJunctionOutliersTable' import { getOtherGeneNames } from '../genes/GeneDetail' -import Transcripts, { ConsequenceDetails, ExtendedSpliceLabel, isManeSelect } from './Transcripts' +import Transcripts, { ConsequenceDetails, isManeSelect } from './Transcripts' import VariantGenes, { GeneLabelContent, omimPhenotypesDetail } from './VariantGene' import { getLocus, @@ -35,6 +35,7 @@ import { } from './VariantUtils' import { GENOME_VERSION_37, GENOME_VERSION_38, getVariantMainTranscript, SVTYPE_LOOKUP, SVTYPE_DETAILS, SCREEN_LABELS, + EXTENDED_INTRONIC_DESCRIPTION, } from '../../../utils/constants' import { camelcaseToTitlecase } from '../../../utils/stringUtils' @@ -639,7 +640,12 @@ const Annotations = React.memo(({ variant, mainGeneId, showMainGene, transcripts )} - + {mainTranscript.spliceregion?.extended_intronic_splice_region_variant && ( +
+ Extended Intronic Splice Region + } content={EXTENDED_INTRONIC_DESCRIPTION} /> +
+ )} {mainTranscript.utrannotator?.fiveutrConsequence && (
UTRAnnotator:   diff --git a/ui/shared/components/panel/variants/Transcripts.jsx b/ui/shared/components/panel/variants/Transcripts.jsx index 0feee268c7..f0e894e127 100644 --- a/ui/shared/components/panel/variants/Transcripts.jsx +++ b/ui/shared/components/panel/variants/Transcripts.jsx @@ -156,19 +156,13 @@ const transcriptIdDetails = (transcript, variant, { transcriptsById, project, up
) -export const ExtendedSpliceLabel = ({ spliceregion }) => (spliceregion?.extended_intronic_splice_region_variant ? ( -