Merge pull request #2208 from jcushman/export-fixes

Add volume_folder to export json
harvard-lil · Mar 6, 2024 · f8db93b · f8db93b
2 parents b9abdf8 + 90768a9
commit f8db93b
Show file tree

Hide file tree

Showing 9 changed files with 23 additions and 8 deletions.
diff --git a/capstone/scripts/export_cap_static.py b/capstone/scripts/export_cap_static.py
@@ -46,7 +46,7 @@ def finalize_reporters_dir(dest_dir: Path) -> None:
         if reporter_dir.name in reporter_slug_dict_reverse:
             reporter = Reporter.objects.get(pk=reporter_slug_dict_reverse[reporter_dir.name])
         else:
-            reporter = Reporter.objects.get(short_name_slug=reporter_dir.name)
+            reporter = Reporter.objects.filter(short_name_slug=reporter_dir.name).exclude(start_year=None).get()
 
         # export reporter metadata
         reporter_dict = call_serializer(ReporterSerializer, reporter)
@@ -57,9 +57,12 @@ def finalize_reporters_dir(dest_dir: Path) -> None:
 
         # write reporter-level VolumesMetadata.json
         print("Writing VolumesMetadata.json")
-        volumes_metadata = [json.loads(f.read_text()) for f in natsorted(reporter_dir.glob("*/VolumeMetadata.json"))]
-        for volume in volumes_metadata:
+        volumes_metadata = []
+        for f in natsorted(reporter_dir.glob("*/VolumeMetadata.json")):
+            volume = json.loads(f.read_text())
+            volume["volume_folder"] = f.parent.name
             volume["reporter_slug"] = reporter_dir.name
+            volumes_metadata.append(volume)
         write_json(reporter_dir / "VolumesMetadata.json", volumes_metadata)
 
     # write ReportersMetadata.json
@@ -114,7 +117,6 @@ def crosscheck_reporters_dir(dest_dir: Path) -> None:
         case_count=Count('case_metadatas', filter=Q(case_metadatas__in_scope=True))
     ).select_related("reporter")
     reporter_dirs = set()
-    # suspicious_volumes = []
     for volume in tqdm(volumes):
         if not volume.case_count:
             continue
@@ -223,6 +225,7 @@ def export_volume(volume: VolumeMetadata, dest_dir: Path) -> None:
     html_dir = temp_volume_dir / "html"
     html_dir.mkdir()
     volume_metadata = volume_to_dict(volume)
+    volume_metadata["volume_folder"] = volume_prefix
     write_json(temp_volume_dir / "VolumeMetadata.json", volume_metadata)
 
     # variables for case export loop

diff --git a/capstone/test_data/cap_static/redacted/VolumesMetadata.json b/capstone/test_data/cap_static/redacted/VolumesMetadata.json
@@ -27,6 +27,7 @@
     "second_part_of_id": null,
     "redacted": true,
     "nominative_reporter": null,
+    "volume_folder": "1",
     "reporter_slug": "us"
   },
   {
@@ -57,6 +58,7 @@
     "second_part_of_id": null,
     "redacted": true,
     "nominative_reporter": null,
+    "volume_folder": "2",
     "reporter_slug": "us"
   }
 ]
diff --git a/capstone/test_data/cap_static/redacted/us/1/VolumeMetadata.json b/capstone/test_data/cap_static/redacted/us/1/VolumeMetadata.json
@@ -25,5 +25,6 @@
   "publication_city": null,
   "second_part_of_id": null,
   "redacted": true,
-  "nominative_reporter": null
+  "nominative_reporter": null,
+  "volume_folder": "1"
 }
diff --git a/capstone/test_data/cap_static/redacted/us/2/VolumeMetadata.json b/capstone/test_data/cap_static/redacted/us/2/VolumeMetadata.json
@@ -25,5 +25,6 @@
   "publication_city": null,
   "second_part_of_id": null,
   "redacted": true,
-  "nominative_reporter": null
+  "nominative_reporter": null,
+  "volume_folder": "2"
 }
diff --git a/capstone/test_data/cap_static/redacted/us/VolumesMetadata.json b/capstone/test_data/cap_static/redacted/us/VolumesMetadata.json
@@ -27,6 +27,7 @@
     "second_part_of_id": null,
     "redacted": true,
     "nominative_reporter": null,
+    "volume_folder": "1",
     "reporter_slug": "us"
   },
   {
@@ -57,6 +58,7 @@
     "second_part_of_id": null,
     "redacted": true,
     "nominative_reporter": null,
+    "volume_folder": "2",
     "reporter_slug": "us"
   }
 ]
diff --git a/capstone/test_data/cap_static/unredacted/VolumesMetadata.json b/capstone/test_data/cap_static/unredacted/VolumesMetadata.json
@@ -27,6 +27,7 @@
     "second_part_of_id": null,
     "redacted": false,
     "nominative_reporter": null,
+    "volume_folder": "1",
     "reporter_slug": "us"
   },
   {
@@ -57,6 +58,7 @@
     "second_part_of_id": null,
     "redacted": false,
     "nominative_reporter": null,
+    "volume_folder": "2",
     "reporter_slug": "us"
   }
 ]
diff --git a/capstone/test_data/cap_static/unredacted/us/1/VolumeMetadata.json b/capstone/test_data/cap_static/unredacted/us/1/VolumeMetadata.json
@@ -25,5 +25,6 @@
   "publication_city": null,
   "second_part_of_id": null,
   "redacted": false,
-  "nominative_reporter": null
+  "nominative_reporter": null,
+  "volume_folder": "1"
 }
diff --git a/capstone/test_data/cap_static/unredacted/us/2/VolumeMetadata.json b/capstone/test_data/cap_static/unredacted/us/2/VolumeMetadata.json
@@ -25,5 +25,6 @@
   "publication_city": null,
   "second_part_of_id": null,
   "redacted": false,
-  "nominative_reporter": null
+  "nominative_reporter": null,
+  "volume_folder": "2"
 }
diff --git a/capstone/test_data/cap_static/unredacted/us/VolumesMetadata.json b/capstone/test_data/cap_static/unredacted/us/VolumesMetadata.json
@@ -27,6 +27,7 @@
     "second_part_of_id": null,
     "redacted": false,
     "nominative_reporter": null,
+    "volume_folder": "1",
     "reporter_slug": "us"
   },
   {
@@ -57,6 +58,7 @@
     "second_part_of_id": null,
     "redacted": false,
     "nominative_reporter": null,
+    "volume_folder": "2",
     "reporter_slug": "us"
   }
 ]