delete old reference data code 😝

download_and_create_reference_datasets/v02/hail_scripts/write_clinvar_ht.py

@@ -3,18 +3,15 @@
 import hail as hl
 
 from v03_pipeline.lib.model import ReferenceGenome
-from v03_pipeline.lib.reference_data.clinvar import (
-    download_and_import_latest_clinvar_vcf,
-    CLINVAR_GOLD_STARS_LOOKUP,
-)
 from hail_scripts.utils.hail_utils import write_ht
+from v03_pipeline.lib.reference_datasets.clinvar import CLINVAR_GOLD_STARS_LOOKUP, get_ht
 
 CLINVAR_PATH = 'ftp://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_{reference_genome}/clinvar.vcf.gz'
 CLINVAR_HT_PATH = 'gs://seqr-reference-data/{reference_genome}/clinvar/clinvar.{reference_genome}.ht'
 
 for reference_genome in ReferenceGenome:
     clinvar_url = CLINVAR_PATH.format(reference_genome=reference_genome.value)
-    ht = download_and_import_latest_clinvar_vcf(clinvar_url, reference_genome)
+    ht = get_ht(clinvar_url, reference_genome)
     timestamp = hl.eval(ht.version)
     ht = ht.annotate(
         gold_stars=CLINVAR_GOLD_STARS_LOOKUP.get(hl.delimit(ht.info.CLNREVSTAT))

v03_pipeline/lib/annotations/fields_test.py

@@ -6,32 +6,39 @@
 from v03_pipeline.lib.annotations.fields import get_fields
 from v03_pipeline.lib.model import (
     DatasetType,
-    ReferenceDatasetCollection,
     ReferenceGenome,
 )
-from v03_pipeline.lib.paths import valid_reference_dataset_collection_path
+from v03_pipeline.lib.paths import valid_reference_dataset_path
+from v03_pipeline.lib.reference_datasets.reference_dataset import ReferenceDataset
 from v03_pipeline.lib.test.mocked_dataroot_testcase import MockedDatarootTestCase
 from v03_pipeline.lib.vep import run_vep
 from v03_pipeline.var.test.vep.mock_vep_data import MOCK_37_VEP_DATA, MOCK_38_VEP_DATA
 
-TEST_INTERVAL_1 = 'v03_pipeline/var/test/reference_data/test_interval_1.ht'
 GRCH37_TO_GRCH38_LIFTOVER_REF_PATH = (
     'v03_pipeline/var/test/liftover/grch37_to_grch38.over.chain.gz'
 )
 GRCH38_TO_GRCH37_LIFTOVER_REF_PATH = (
     'v03_pipeline/var/test/liftover/grch38_to_grch37.over.chain.gz'
 )
+TEST_GNOMAD_NONCODING_CONSTRAINT_38_HT = 'v03_pipeline/var/test/reference_datasets/GRCh38/gnomad_non_coding_constraint/1.0.ht'
+TEST_SCREEN_38_HT = 'v03_pipeline/var/test/reference_datasets/GRCh38/screen/1.0.ht'
 
 
 class FieldsTest(MockedDatarootTestCase):
     def setUp(self) -> None:
         super().setUp()
         shutil.copytree(
-            TEST_INTERVAL_1,
-            valid_reference_dataset_collection_path(
+            TEST_GNOMAD_NONCODING_CONSTRAINT_38_HT,
+            valid_reference_dataset_path(
                 ReferenceGenome.GRCh38,
-                DatasetType.SNV_INDEL,
-                ReferenceDatasetCollection.INTERVAL,
+                ReferenceDataset.gnomad_non_coding_constraint,
+            ),
+        )
+        shutil.copytree(
+            TEST_SCREEN_38_HT,
+            valid_reference_dataset_path(
+                ReferenceGenome.GRCh38,
+                ReferenceDataset.screen,
             ),
         )
 
@@ -120,18 +127,17 @@ def test_get_formatting_fields(self, mock_vep: Mock) -> None:
                             reference_genome,
                         ),
                         **{
-                            f'{rdc.value}_ht': hl.read_table(
-                                valid_reference_dataset_collection_path(
+                            f'{reference_dataset}_ht': hl.read_table(
+                                valid_reference_dataset_path(
                                     reference_genome,
-                                    DatasetType.SNV_INDEL,
-                                    rdc,
+                                    reference_dataset,
                                 ),
                             )
-                            for rdc in ReferenceDatasetCollection.for_reference_genome_dataset_type(
+                            for reference_dataset in ReferenceDataset.for_reference_genome_dataset_type_annotations(
                                 reference_genome,
                                 DatasetType.SNV_INDEL,
                             )
-                            if rdc.requires_annotation
+                            if reference_dataset.is_keyed_by_interval
                         },
                         **(
                             {

v03_pipeline/lib/annotations/mito.py

@@ -47,14 +47,6 @@ def HL(mt: hl.MatrixTable, **_: Any) -> hl.Expression:  # noqa: N802
     return hl.if_else(is_called, mt.HL, 0)
 
 
-def high_constraint_region_mito(
-    ht: hl.Table,
-    interval_ht: hl.Table,
-    **_: Any,
-) -> hl.Expression:
-    return hl.is_defined(interval_ht[ht.locus])
-
-
 def mito_cn(mt: hl.MatrixTable, **_: Any) -> hl.Expression:
     return hl.int32(mt.mito_cn)
 

v03_pipeline/lib/annotations/snv_indel.py

@@ -73,16 +73,16 @@ def gt_stats(
 
 def gnomad_non_coding_constraint(
     ht: hl.Table,
-    interval_ht: hl.Table,
+    gnomad_non_coding_constraint_ht: hl.Table,
     **_: Any,
 ) -> hl.Expression:
     return hl.Struct(
         z_score=(
-            interval_ht.index(ht.locus, all_matches=True)
+            gnomad_non_coding_constraint_ht.index(ht.locus, all_matches=True)
             .filter(
-                lambda x: hl.is_defined(x.gnomad_non_coding_constraint['z_score']),
+                lambda x: hl.is_defined(x['z_score']),
             )
-            .gnomad_non_coding_constraint.z_score.first()
+            .z_score.first()
         ),
     )
 
@@ -98,16 +98,16 @@ def rg38_locus(
 
 def screen(
     ht: hl.Table,
-    interval_ht: hl.Table,
+    screen_ht: hl.Table,
     **_: Any,
 ) -> hl.Expression:
     return hl.Struct(
         region_type_ids=(
-            interval_ht.index(
+            screen_ht.index(
                 ht.locus,
                 all_matches=True,
             ).flatmap(
-                lambda x: x.screen['region_type_ids'],
+                lambda x: x['region_type_ids'],
             )
         ),
     )

v03_pipeline/lib/model/__init__.py

@@ -1,6 +1,3 @@
-from v03_pipeline.lib.model.cached_reference_dataset_query import (
-    CachedReferenceDatasetQuery,
-)
 from v03_pipeline.lib.model.dataset_type import DatasetType
 from v03_pipeline.lib.model.definitions import (
     AccessControl,
@@ -10,18 +7,13 @@
     Sex,
 )
 from v03_pipeline.lib.model.environment import Env
-from v03_pipeline.lib.model.reference_dataset_collection import (
-    ReferenceDatasetCollection,
-)
 
 __all__ = [
     'AccessControl',
-    'CachedReferenceDatasetQuery',
     'DatasetType',
     'Env',
     'Sex',
     'PipelineVersion',
-    'ReferenceDatasetCollection',
     'ReferenceGenome',
     'SampleType',
 ]

v03_pipeline/lib/model/dataset_type.py

@@ -246,7 +246,6 @@ def formatting_annotation_fns(
             DatasetType.MITO: [
                 mito.common_low_heteroplasmy,
                 mito.haplogroup,
-                mito.high_constraint_region_mito,
                 mito.mitotip,
                 mito.rsid,
                 shared.variant_id,