add a checkpoint for an annoying issue

broadinstitute · Nov 27, 2024 · 7f372e8 · 7f372e8
1 parent 0ad6c1b
commit 7f372e8
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/v03_pipeline/lib/reference_datasets/dbnsfp.py b/v03_pipeline/lib/reference_datasets/dbnsfp.py
@@ -65,6 +65,9 @@ def get_ht(path: str, reference_genome: ReferenceGenome) -> hl.Table:
             missing='.',
             force=True,
         )
+        # NB: We ran into weird issues...running out
+        # of file descriptors on dataproc :/
+        ht = checkpoint(ht)
         select_fields = {'ref', 'alt', *types.keys(), *rename.keys()}
         ht = ht.select(
             *select_fields,