WIP import data from datasources for use in mapping

hub/graphql/mutations.py

@@ -8,13 +8,14 @@
 from strawberry_django.permissions import IsAuthenticated
 from strawberry_django.auth.utils import get_current_user
 from strawberry.types.info import Info
+from asgiref.sync import async_to_sync
 
 @strawberry.input
 class IDObject:
     id: str
 
 @strawberry.input
-class AutoUpdateMappingItemInput:
+class UpdateMappingItemInput:
     source: str
     source_path: str
     destination_column: str
@@ -28,7 +29,7 @@ class ExternalDataSourceInput:
     geography_column: auto
     geography_column_type: auto
     auto_update_enabled: auto
-    auto_update_mapping: Optional[List[AutoUpdateMappingItemInput]]
+    update_mapping: Optional[List[UpdateMappingItemInput]]
     auto_import_enabled: auto
 
 @strawberry_django.input(models.AirtableSource, partial=True)
@@ -66,7 +67,10 @@ def disable_auto_update (external_data_source_id: str) -> models.ExternalDataSou
 @strawberry.mutation(extensions=[IsAuthenticated()])
 def trigger_update(external_data_source_id: str) -> models.ExternalDataSource:
     data_source = models.ExternalDataSource.objects.get(id=external_data_source_id)
-    job_id = data_source.schedule_refresh_all()
+    # TODO: Return this to the queue
+    print("Triggering update")
+    async_to_sync(data_source.refresh_all)()
+    # job_id = data_source.schedule_refresh_all()
     return data_source
 
 @strawberry.mutation(extensions=[IsAuthenticated()])

hub/graphql/types.py

@@ -6,6 +6,7 @@
 from hub import models
 from datetime import datetime
 import procrastinate.contrib.django.models
+from .utils import key_resolver
 
 @strawberry_django.filters.filter(procrastinate.contrib.django.models.ProcrastinateJob, lookups=True)
 class QueueFilter:
@@ -96,6 +97,12 @@ def get_queryset(cls, queryset, info, **kwargs):
 
 # ExternalDataSource
 
+@strawberry.type
+class FieldDefinition:
+    value: str = key_resolver('value')
+    label: Optional[str] = key_resolver('label')
+    description: Optional[str] = key_resolver('description')
+
 @strawberry_django.type(models.ExternalDataSource)
 class ExternalDataSource:
     id: auto
@@ -106,9 +113,12 @@ class ExternalDataSource:
     organisation: Organisation
     geography_column: auto
     geography_column_type: auto
-    auto_update_mapping: Optional[List['AutoUpdateConfig']]
+    update_mapping: Optional[List['AutoUpdateConfig']]
     auto_update_enabled: auto
     auto_import_enabled: auto
+    field_definitions: Optional[List[FieldDefinition]] = strawberry_django.field(
+        resolver=lambda self: self.field_definitions()
+    )
 
     jobs: List[QueueJob] = strawberry_django.field(
         resolver=lambda self: procrastinate.contrib.django.models.ProcrastinateJob.objects.filter(

hub/graphql/utils.py

@@ -0,0 +1,6 @@
+import strawberry
+
+def key_resolver(key: str):
+    def resolver(self):
+        return self.get(key, None)
+    return strawberry.field(resolver=resolver)

hub/migrations/0075_alter_externaldatasource_geography_column_type.py

@@ -22,7 +22,7 @@ class Migration(migrations.Migration):
                     ("council", "Council"),
                     ("constituency", "Constituency"),
                 ],
-                choices_enum=hub.models.ExternalDataSource.GeographyTypes,
+                choices_enum=hub.models.ExternalDataSource.PostcodesIOGeographyTypes,
                 default="postcode",
                 max_length=12,
             ),

hub/migrations/0077_externaldatasource_fields_and_more.py

@@ -0,0 +1,28 @@
+# Generated by Django 4.2.10 on 2024-03-11 03:13
+
+from django.db import migrations
+import django_jsonform.models.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("hub", "0076_externaldatasource_autoimport_enabled_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="externaldatasource",
+            name="fields",
+            field=django_jsonform.models.fields.JSONField(
+                blank=True, default=[], null=True
+            ),
+        ),
+        migrations.AlterField(
+            model_name="externaldatasource",
+            name="auto_update_mapping",
+            field=django_jsonform.models.fields.JSONField(
+                blank=True, default=[], null=True
+            ),
+        ),
+    ]

hub/migrations/0078_rename_auto_update_mapping_externaldatasource_update_mapping.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.10 on 2024-03-11 03:15
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("hub", "0077_externaldatasource_fields_and_more"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="externaldatasource",
+            old_name="auto_update_mapping",
+            new_name="update_mapping",
+        ),
+    ]

hub/migrations/0079_alter_externaldatasource_geography_column_type.py

@@ -0,0 +1,31 @@
+# Generated by Django 4.2.10 on 2024-03-11 03:45
+
+from django.db import migrations
+import django_choices_field.fields
+import hub.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("hub", "0078_rename_auto_update_mapping_externaldatasource_update_mapping"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="externaldatasource",
+            name="geography_column_type",
+            field=django_choices_field.fields.TextChoicesField(
+                choices=[
+                    ("postcode", "Postcode"),
+                    ("ward", "Ward"),
+                    ("council", "Council"),
+                    ("constituency", "Constituency"),
+                    ("constituency_2025", "Constituency (2024)"),
+                ],
+                choices_enum=hub.models.ExternalDataSource.PostcodesIOGeographyTypes,
+                default="postcode",
+                max_length=17,
+            ),
+        ),
+    ]

hub/migrations/0080_dataset_external_data_source_and_more.py

@@ -0,0 +1,39 @@
+# Generated by Django 4.2.10 on 2024-03-11 09:56
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_jsonform.models.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("hub", "0079_alter_externaldatasource_geography_column_type"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="dataset",
+            name="external_data_source",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                to="hub.externaldatasource",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="externaldatasource",
+            name="fields",
+            field=django_jsonform.models.fields.JSONField(
+                blank=True, default=list, null=True
+            ),
+        ),
+        migrations.AlterField(
+            model_name="externaldatasource",
+            name="update_mapping",
+            field=django_jsonform.models.fields.JSONField(
+                blank=True, default=list, null=True
+            ),
+        ),
+    ]