Distinguish Member Lists and Custom Data Layers so that you can't piv…

…ot a member list by another member list

hub/graphql/mutations.py

@@ -24,6 +24,7 @@ class UpdateMappingItemInput:
 class ExternalDataSourceInput:
     id: auto
     name: auto
+    data_type: auto
     description: auto
     organisation: auto
     geography_column: auto

hub/graphql/types.py

@@ -103,10 +103,16 @@ class FieldDefinition:
     label: Optional[str] = key_resolver('label')
     description: Optional[str] = key_resolver('description')
 
-@strawberry_django.type(models.ExternalDataSource)
+@strawberry_django.filter(models.ExternalDataSource)
+class ExternalDataSourceFilter:
+    data_type: auto
+    geography_column_type: auto
+
+@strawberry_django.type(models.ExternalDataSource, filters=ExternalDataSourceFilter)
 class ExternalDataSource:
     id: auto
     name: auto
+    data_type: auto
     description: auto
     created_at: auto
     last_update: auto

hub/migrations/0083_externaldatasource_data_type.py

@@ -0,0 +1,29 @@
+# Generated by Django 4.2.10 on 2024-03-13 01:37
+
+from django.db import migrations
+import django_choices_field.fields
+import hub.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("hub", "0082_alter_externaldatasource_geography_column_type"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="externaldatasource",
+            name="data_type",
+            field=django_choices_field.fields.TextChoicesField(
+                choices=[
+                    ("member", "Members or supporters"),
+                    ("region", "Areas or regions"),
+                    ("other", "Other"),
+                ],
+                choices_enum=hub.models.ExternalDataSource.DataSourceType,
+                default="other",
+                max_length=6,
+            ),
+        ),
+    ]

hub/models.py

@@ -781,6 +781,11 @@ class ExternalDataSource(PolymorphicModel):
     '''
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     organisation = models.ForeignKey(Organisation, on_delete=models.CASCADE, related_name='external_data_sources', null=True, blank=True)
+    class DataSourceType(models.TextChoices):
+        MEMBER = 'member', 'Members or supporters'
+        REGION = 'region', 'Areas or regions'
+        OTHER = 'other', 'Other'
+    data_type = TextChoicesField(choices_enum=DataSourceType, default=DataSourceType.OTHER)
     name = models.CharField(max_length=250)
     description = models.TextField(blank=True, null=True)
     created_at = models.DateTimeField(auto_now_add=True)
@@ -1073,14 +1078,16 @@ async def map_one(self, member: Union[str, dict], loaders: Loaders) -> MappedMem
                     if postcode_data is not None:
                         update_fields[destination_column] = get(postcode_data, source_path)
                 else:
-                    update_fields[destination_column] = await loaders['source_loaders'][source].load(
-                        self.EnrichmentLookup(
-                            member_id=self.get_record_id(member),
-                            postcode_data=postcode_data,
-                            source_id=source,
-                            source_path=source_path
+                    source_loader = loaders['source_loaders'].get(source, None)
+                    if source_loader is not None:
+                        update_fields[destination_column] = await source_loader.load(
+                            self.EnrichmentLookup(
+                                member_id=self.get_record_id(member),
+                                postcode_data=postcode_data,
+                                source_id=source,
+                                source_path=source_path
+                            )
                         )
-                    )
             # Return the member and config data
             return self.MappedMember(
                 member=member,
@@ -1133,7 +1140,9 @@ def enable_auto_update(self) -> Union[None, int]:
         if self.automated_webhooks:
             self.refresh_webhooks()
             # And schedule a cron to keep doing it
-            refresh_webhooks.defer()
+            refresh_webhooks.defer(
+                external_data_source_id=str(self.id),
+            )
         self.auto_update_enabled = True
         self.save()
 

nextjs/package.json

@@ -37,6 +37,7 @@
     "mapbox-gl": "^3.2.0",
     "next": "14.1.0",
     "next-themes": "^0.2.1",
+    "query-string": "^9.0.0",
     "react": "^18",
     "react-dom": "^18",
     "react-hook-form": "^7.51.0",