Dev

.github/workflows/prod-hail-search-release.yaml

@@ -45,11 +45,28 @@ jobs:
           persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token
           fetch-depth: 0 # otherwise, you will failed to push refs to dest repo
 
-      - name: Update appVersion in hail-search Chart file
+      - name: Get latest hail-search version
         uses: mikefarah/yq@v4.22.1
+        id: current
         with:
           cmd: >
-            yq -i '.appVersion = "${{ github.event.workflow_run.head_sha }}"' charts/hail-search/Chart.yaml
+            yq -r '.version' charts/hail-search/Chart.yaml
+
+      - name: Bump version
+        id: bump
+        uses: cbrgm/semver-bump-action@main
+        with:
+          current-version: ${{ steps.current.outputs.result }}
+          bump-level: minor
+
+      - name: Update appVersion and version in seqr Chart file
+        uses: mikefarah/yq@v4.22.1
+        with:
+          cmd: >
+            yq -i '
+            .appVersion = "${{ github.event.workflow_run.head_sha }}" |
+            .version = "${{ steps.bump.outputs.new_version }}"
+            ' charts/hail-search/Chart.yaml
 
       - name: Commit and Push changes
         uses: Andro999b/push@v1.3

.github/workflows/prod-release.yaml

@@ -45,11 +45,28 @@ jobs:
           persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token
           fetch-depth: 0 # otherwise, you will failed to push refs to dest repo
 
-      - name: Update appVersion in seqr Chart file
+      - name: Get latest seqr version
         uses: mikefarah/yq@v4.22.1
+        id: current
         with:
           cmd: >
-            yq -i '.appVersion = "${{ github.event.workflow_run.head_sha }}"' charts/seqr/Chart.yaml
+            yq -r '.version' charts/seqr/Chart.yaml
+
+      - name: Bump version
+        id: bump
+        uses: cbrgm/semver-bump-action@main
+        with:
+          current-version: ${{ steps.current.outputs.result }}
+          bump-level: minor
+
+      - name: Update appVersion and version in seqr Chart file
+        uses: mikefarah/yq@v4.22.1
+        with:
+          cmd: >
+            yq -i '
+            .appVersion = "${{ github.event.workflow_run.head_sha }}" |
+            .version = "${{ steps.bump.outputs.new_version }}"
+            ' charts/hail-search/Chart.yaml
 
       - name: Commit and Push changes
         uses: Andro999b/push@v1.3

deploy/LOCAL_INSTALL_HELM.md

@@ -76,7 +76,7 @@ loading_pipeline_queue  test.vcf.gz
 ```
 - In the top header of *seqr*, click on the **Data Management** button.
 - In the subheader, click on **Load Data**.
-- Type the name of the callset path into the **Callset File Path** text box (without the directory prefix), and select the appropriate Sample Type (WES/WGS) and Genome Version (GRCh37/GRCh38) for your project.  The pipeline includes a sequence of validation steps to insure the validity of your VCF, but these may be skipped by enabling the **Skip Callset Validation**option.  We strongly recommend leaving validation enabled to ensure the quality of your analysis.
+- Select your VCF from the dropdown and select the appropriate Sample Type (WES/WGS) and Genome Version (GRCh37/GRCh38) for your project.  The pipeline includes a sequence of validation steps to insure the validity of your VCF, but these may be skipped by enabling the **Skip Callset Validation**option.  We strongly recommend leaving validation enabled to ensure the quality of your analysis.
 - Click through to the next page and select your project from the **Projects to Load** dropdown, then click **Submit**.
 - If you wish to check the status of the loading request, you can click through to the **Pipeline Status** tab to view the loading pipeline interface.
 - Data should be loaded into the search backend automatically, usually within a few hours.

hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/README.txt

@@ -1,3 +1,3 @@
 This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
   Written with version 0.2.128-eead8100a1c1
-  Created at 2024/08/16 15:39:04
+  Created at 2025/01/02 17:23:59

hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/README.txt

@@ -0,0 +1,3 @@
+This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
+  Written with version 0.2.128-eead8100a1c1
+  Created at 2025/01/02 17:57:36

hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/README.txt

@@ -1,3 +1,3 @@
 This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
-  Written with version 0.2.130-bea04d9c79b5
-  Created at 2024/11/04 13:45:23
+  Written with version 0.2.133-4c60fddb171a
+  Created at 2025/01/02 12:26:15

hail_search/queries/base.py

@@ -85,6 +85,9 @@ class BaseHailTableQuery(object):
 
     SORTS = {
         XPOS: lambda r: [r.xpos],
+        'family_guid': lambda r: [
+            hl.int(r.family_entries.find(hl.is_defined).first().familyGuid.first_match_in('(\d+)').first())
+        ],
     }
 
     @classmethod
@@ -1222,7 +1225,7 @@ def gene_counts(self):
     def _filter_variant_ids(self, ht, variant_ids):
         return ht
 
-    def lookup_variants(self, variant_ids):
+    def lookup_variants(self, variant_ids, additional_annotations=None):
         self._parse_intervals(intervals=None, variant_ids=variant_ids, variant_keys=variant_ids)
         ht = self._read_table('annotations.ht', drop_globals=['versions'])
         ht = self._filter_variant_ids(ht, variant_ids)
@@ -1232,6 +1235,8 @@ def lookup_variants(self, variant_ids):
             k: v for k, v in self.annotation_fields(include_genotype_overrides=False).items()
             if k not in {FAMILY_GUID_FIELD, GENOTYPES_FIELD}
         }
+        if additional_annotations:
+            annotation_fields.update(additional_annotations)
         formatted = self._format_results(ht.key_by(), annotation_fields=annotation_fields, include_genotype_overrides=False)
 
         return formatted.aggregate(hl.agg.take(formatted.row, len(variant_ids)))
@@ -1246,9 +1251,13 @@ def _get_variant_project_data(self, variant_id, **kwargs):
         return project_data[0] if project_data else {}
 
     def lookup_variant(self, variant_id, **kwargs):
-        variants = self.lookup_variants([variant_id])
+        variants = self.lookup_variants([variant_id], additional_annotations=self._lookup_variant_annotations())
         if not variants:
             raise HTTPNotFound()
         variant = dict(variants[0])
-        variant.update(self._get_variant_project_data(variant_id, **kwargs))
+        variant.update(self._get_variant_project_data(variant_id, variant=variant, **kwargs))
         return variant
+
+    @staticmethod
+    def _lookup_variant_annotations():
+        return {}

hail_search/queries/snv_indel.py

@@ -2,7 +2,7 @@
 import hail as hl
 
 from hail_search.constants import GENOME_VERSION_GRCh38, SCREEN_KEY, PREFILTER_FREQ_CUTOFF, ALPHAMISSENSE_SORT, \
-    UTR_ANNOTATOR_KEY, EXTENDED_SPLICE_KEY, MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY
+    UTR_ANNOTATOR_KEY, EXTENDED_SPLICE_KEY, MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY, GENOME_VERSION_GRCh37
 from hail_search.queries.base import BaseHailTableQuery, PredictionPath
 from hail_search.queries.snv_indel_37 import SnvIndelHailTableQuery37
 
@@ -12,6 +12,7 @@
 class SnvIndelHailTableQuery(SnvIndelHailTableQuery37):
 
     GENOME_VERSION = GENOME_VERSION_GRCh38
+    LIFT_GENOME_VERSION = GENOME_VERSION_GRCh37
     PREDICTION_FIELDS_CONFIG = {
         **SnvIndelHailTableQuery37.PREDICTION_FIELDS_CONFIG,
         'fathmm': PredictionPath('dbnsfp', 'fathmm_MKL_coding_score'),
@@ -96,3 +97,7 @@ def _get_annotation_override_filters(self, ht, annotation_overrides):
                 )
 
         return annotation_filters
+
+    @staticmethod
+    def _lookup_variant_annotations():
+        return {'liftover_locus': lambda r: r.rg37_locus}

hail_search/queries/snv_indel_37.py

@@ -1,9 +1,10 @@
+from aiohttp.web import HTTPNotFound
 from collections import OrderedDict
 import hail as hl
 
 from hail_search.constants import CLINVAR_KEY, HGMD_KEY, HGMD_PATH_RANGES, \
     GNOMAD_GENOMES_FIELD, PREFILTER_FREQ_CUTOFF, PATH_FREQ_OVERRIDE_CUTOFF, PATHOGENICTY_HGMD_SORT_KEY, \
-    SPLICE_AI_FIELD, GENOME_VERSION_GRCh37
+    SPLICE_AI_FIELD, GENOME_VERSION_GRCh37, GENOME_VERSION_GRCh38
 from hail_search.queries.base import PredictionPath, QualityFilterFormat
 from hail_search.queries.mito import MitoHailTableQuery
 
@@ -12,6 +13,7 @@ class SnvIndelHailTableQuery37(MitoHailTableQuery):
 
     DATA_TYPE = 'SNV_INDEL'
     GENOME_VERSION = GENOME_VERSION_GRCh37
+    LIFT_GENOME_VERSION = GENOME_VERSION_GRCh38
 
     GENOTYPE_FIELDS = {f.lower(): f for f in ['DP', 'GQ', 'AB']}
     QUALITY_FILTER_FORMAT = {
@@ -133,3 +135,26 @@ def _get_annotation_override_filters(self, ht, annotation_overrides):
     @staticmethod
     def _stat_has_non_ref(s):
         return (s.het_samples > 0) | (s.hom_samples > 0)
+
+    @staticmethod
+    def _lookup_variant_annotations():
+        return {'liftover_locus': lambda r: r.rg38_locus}
+
+    @classmethod
+    def _get_lifted_table_path(cls, path):
+        return f'{cls._get_table_dir(path)}/{cls.LIFT_GENOME_VERSION}/{cls.DATA_TYPE}/{path}'
+
+    def _get_variant_project_data(self, variant_id, variant=None, **kwargs):
+        project_data = super()._get_variant_project_data(variant_id, **kwargs)
+        liftover_locus = variant.pop('liftover_locus')
+        if not liftover_locus:
+            return project_data
+        interval = hl.eval(hl.interval(liftover_locus, liftover_locus, includes_start=True, includes_end=True))
+        self._load_table_kwargs['_intervals'] = [interval]
+        self._get_table_path = self._get_lifted_table_path
+        try:
+            lift_project_data = super()._get_variant_project_data(variant_id, **kwargs)
+        except HTTPNotFound:
+            return project_data
+        project_data['familyGenotypes'].update(lift_project_data['familyGenotypes'])
+        return project_data.annotate(liftedFamilyGuids=sorted(lift_project_data['familyGenotypes'].keys()))

hail_search/test_search.py

@@ -14,9 +14,9 @@
     EXPECTED_SAMPLE_DATA_WITH_SEX, SV_WGS_SAMPLE_DATA_WITH_SEX, VARIANT_LOOKUP_VARIANT, \
     MULTI_PROJECT_SAMPLE_TYPES_SAMPLE_DATA, FAMILY_2_BOTH_SAMPLE_TYPE_SAMPLE_DATA, \
     VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, FAMILY_2_BOTH_SAMPLE_TYPE_SAMPLE_DATA_MISSING_PARENTAL_WGS, \
-    VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY, \
+    VARIANT4_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY, \
     VARIANT1_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY, VARIANT3_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY, \
-    VARIANT4_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY
+    VARIANT4_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY, VARIANT4_WES_ONLY, VARIANT3_WES_ONLY
 from hail_search.web_app import init_web_app, sync_to_async_hail_query
 from hail_search.queries.base import BaseHailTableQuery
 
@@ -370,10 +370,11 @@ async def test_both_sample_types_search(self):
 
         # Variant 1 is de novo in exome but inherited and homozygous in genome.
         # Variant 2 is inherited and homozygous in exome and de novo and homozygous in genome.
-        # Variant 3 is inherited in both sample types. Variant 4 is de novo in both sample types.
+        # Variant 3 is inherited in both sample types.
+        # Variant 4 is de novo in exome, but inherited in genome in the same parent that has variant 3.
         inheritance_mode = 'recessive'
         await self._assert_expected_search(
-            [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, [VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES]],
+            [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, [VARIANT3_WES_ONLY, VARIANT4_WES_ONLY]],
             sample_data=FAMILY_2_BOTH_SAMPLE_TYPE_SAMPLE_DATA, inheritance_mode=inheritance_mode,
             **COMP_HET_ALL_PASS_FILTERS
         )
@@ -1241,6 +1242,12 @@ async def test_sort(self):
             sort_metadata={'ENSG00000177000': 3},
         )
 
+        await self._assert_expected_search(
+            [_sorted(MULTI_PROJECT_VARIANT1, [2]), _sorted(MULTI_PROJECT_VARIANT2, [2]),
+             _sorted(VARIANT3, [2]), _sorted(VARIANT4, [2]), _sorted(PROJECT_2_VARIANT, [11])],
+            sort='family_guid', sample_data=MULTI_PROJECT_SAMPLE_DATA,
+        )
+
         # size sort only applies to SVs, so has no impact on other variant
         await self._assert_expected_search(
             [_sorted(GCNV_VARIANT1, [-171766]), _sorted(GCNV_VARIANT2, [-17768]), _sorted(GCNV_VARIANT4, [-14487]),

hail_search/test_utils.py

@@ -380,27 +380,19 @@
     'CAID': 'CA10960369',
 }
 
-VARIANT3_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY = deepcopy(VARIANT3)
-genotypes = VARIANT3_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY['genotypes']
-VARIANT3_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY['genotypes'] = {
-    'I000004_hg00731': [
-        genotypes['I000004_hg00731'],
-        {**genotypes['I000004_hg00731'], 'sampleType': 'WGS'}
-    ],
+VARIANT3_WES_ONLY = deepcopy(VARIANT3)
+genotypes = VARIANT3_WES_ONLY['genotypes']
+VARIANT3_WES_ONLY['genotypes'] = {
+    'I000004_hg00731': [genotypes['I000004_hg00731']],
     'I000005_hg00732': [genotypes['I000005_hg00732']],
     'I000006_hg00733': [genotypes['I000006_hg00733']],
 }
 
-VARIANT3_BOTH_SAMPLE_TYPES = deepcopy(VARIANT3_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY)
-genotypes = VARIANT3_BOTH_SAMPLE_TYPES['genotypes']
-VARIANT3_BOTH_SAMPLE_TYPES['genotypes']['I000005_hg00732'] = [
-    *genotypes['I000005_hg00732'],
-    {**genotypes['I000005_hg00732'][0], 'sampleType': 'WGS'}
-]
-VARIANT3_BOTH_SAMPLE_TYPES['genotypes']['I000006_hg00733'] = [
-    *genotypes['I000006_hg00733'],
-    {**genotypes['I000006_hg00733'][0], 'sampleType': 'WGS'}
-]
+VARIANT3_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY = deepcopy(VARIANT3_WES_ONLY)
+genotypes = VARIANT3_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY['genotypes']
+VARIANT3_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY['genotypes']['I000004_hg00731'].append(
+    {**genotypes['I000004_hg00731'][0], 'sampleType': 'WGS'}
+)
 
 VARIANT4 = {
     'variantId': '1-91511686-T-G',
@@ -475,17 +467,20 @@
     'CAID': 'CA341062623',
 }
 
-VARIANT4_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY = deepcopy(VARIANT4)
-genotypes = VARIANT4_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY['genotypes']
-VARIANT4_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY['genotypes'] = {
-    'I000004_hg00731': [
-        genotypes['I000004_hg00731'],
-        {**genotypes['I000004_hg00731'], 'sampleType': 'WGS'}
-    ],
+VARIANT4_WES_ONLY = deepcopy(VARIANT4)
+genotypes = VARIANT4_WES_ONLY['genotypes']
+VARIANT4_WES_ONLY['genotypes'] = {
+    'I000004_hg00731': [genotypes['I000004_hg00731']],
     'I000005_hg00732': [genotypes['I000005_hg00732']],
     'I000006_hg00733': [genotypes['I000006_hg00733']],
 }
 
+VARIANT4_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY = deepcopy(VARIANT4_WES_ONLY)
+genotypes = VARIANT4_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY['genotypes']
+VARIANT4_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY['genotypes']['I000004_hg00731'].append(
+    {**genotypes['I000004_hg00731'][0], 'sampleType': 'WGS'}
+)
+
 VARIANT4_BOTH_SAMPLE_TYPES = deepcopy(VARIANT4_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY)
 genotypes = VARIANT4_BOTH_SAMPLE_TYPES['genotypes']
 VARIANT4_BOTH_SAMPLE_TYPES['genotypes']['I000005_hg00732'] = [
@@ -494,11 +489,12 @@
 ]
 VARIANT4_BOTH_SAMPLE_TYPES['genotypes']['I000006_hg00733'] = [
     *genotypes['I000006_hg00733'],
-    {**genotypes['I000006_hg00733'][0], 'sampleType': 'WGS'}
+    {**genotypes['I000006_hg00733'][0], 'numAlt': 2, 'sampleType': 'WGS'}
 ]
 
 VARIANT_LOOKUP_VARIANT = {
     **VARIANT1,
+    'liftedFamilyGuids': ['F000014_14'],
     'familyGenotypes': {
         VARIANT1['familyGuids'][0]: sorted([
             {k: v for k, v in g.items() if k != 'individualGuid'} for g in VARIANT1['genotypes'].values()
@@ -507,6 +503,10 @@
             'sampleId': 'NA20885', 'sampleType': 'WES', 'familyGuid': 'F000011_11',
             'numAlt': 2, 'dp': 6, 'gq': 16, 'ab': 1.0, 'filters': [],
         }],
+        'F000014_14': [{
+            'sampleId': 'NA21234', 'sampleType': 'WGS', 'familyGuid': 'F000014_14',
+            'numAlt': 1, 'dp': 27, 'gq': 87, 'ab': 0.531000018119812, 'filters': None,
+        }],
     }
 }
 for k in {'familyGuids', 'genotypes'}:

seqr/fixtures/variant_tag_types.json

@@ -480,7 +480,7 @@
         "project": null,
         "name": "Submit to Clinvar",
         "category": "Data Sharing",
-        "description": "By selecting this tag, you are notifying CMG staff that this variant should be submitted to ClinVar. Generally, this is for pathogenic or likely pathogenic variants in known disease genes or for any benign or likely benign variants that are incorrectly annotated in ClinVar. Please also add a note that describes supporting evidence for how you interpreted this variant.",
+        "description": "",
         "color": "#8A62AE",
         "order": 25.0
     }

seqr/management/commands/check_for_new_samples_from_pipeline.py

@@ -166,6 +166,11 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers
         failed_families_by_guid = {f['guid']: f for f in Family.objects.filter(
             guid__in={family for families in failed_family_samples.values() for family in families}
         ).values('guid', 'family_id', 'project__name')}
+        if failed_families_by_guid:
+            Family.bulk_update(
+                user=None, update_json={'analysis_status': Family.ANALYSIS_STATUS_LOADING_FAILED},
+                guid__in=failed_families_by_guid, analysis_status=Family.ANALYSIS_STATUS_WAITING_FOR_DATA
+            )
         failures_by_project_check = defaultdict(lambda: defaultdict(list))
         for check, check_failures in failed_family_samples.items():
             for family_guid, failure_data in check_failures.items():