Merge pull request #166 from commonknowledge/feature/map-668-hnh-ward…

…-csv-issues

Complex geocoding config [MAP-706] [MAP-708] [MAP-707] [MAP-458]

.github/workflows/tests.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     environment: testing
     services:
-      postgres:
+      db:
         image: kartoza/postgis:13
         env:
           POSTGRES_USER: postgres
@@ -31,7 +31,7 @@ jobs:
       # https://stackoverflow.com/questions/78593700/langchain-community-langchain-packages-giving-error-missing-1-required-keywor
       image: python:3.12.3
       env:
-        DATABASE_URL: postgis://postgres:password@postgres:5432/local-intelligence
+        DATABASE_URL: postgis://postgres:password@db:5432/local-intelligence
         CACHE_FILE: /tmp/meep
         POETRY_VIRTUALENVS_CREATE: "false"
         GOOGLE_MAPS_API_KEY: ${{ secrets.GOOGLE_MAPS_API_KEY }}
@@ -63,9 +63,11 @@ jobs:
         run: |
           curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null
           echo "deb https://ngrok-agent.s3.amazonaws.com buster main" | tee /etc/apt/sources.list.d/ngrok.list
-          apt-get update && apt-get install -y binutils gdal-bin libproj-dev ngrok less
+          apt-get update && apt-get install -y binutils gdal-bin libproj-dev ngrok less postgresql-client
       - name: Install poetry
-        run: curl -sSL https://install.python-poetry.org | python3 -
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+          ~/.local/bin/poetry self add poetry-plugin-export
       - name: Install python dependencies
         run: ~/.local/bin/poetry export --with dev --without-hashes -f requirements.txt --output requirements.txt && pip install -r requirements.txt
       - name: Start ngrok tunnelling
@@ -86,6 +88,10 @@ jobs:
         run: gunicorn local_intelligence_hub.asgi:application -k uvicorn.workers.UvicornWorker --bind 0.0.0.0:8000 > server.log 2>&1 &
       - name: Run django tests
         run: cat .env && coverage run --source=. --branch manage.py test || (cat server.log && exit 1)
+      - name: Run geocoding tests in isolation
+        run: |
+          echo "RUN_GEOCODING_TESTS=1" >> .env
+          cat .env && python manage.py test hub.tests.test_external_data_source_parsers || (cat server.log && exit 1)
       - name: Generate coverage xml
         run: coverage xml
       - name: Upload coverage.xml

.gitignore

@@ -7,6 +7,7 @@ __pycache__
 .coverage
 .media
 data/**/*
+!data/areas.psql.zip
 !data/.gitkeep
 !data/areas.psql.zip
 .next

Dockerfile

@@ -12,7 +12,7 @@ ENV INSIDE_DOCKER=1 \
     SHELL=/bin/bash
 RUN curl -sL https://deb.nodesource.com/setup_22.x | bash
 RUN apt-get update && apt-get install -y \
-    binutils gdal-bin libproj-dev git nodejs python3-dev \
+    binutils gdal-bin libproj-dev git nodejs python3-dev postgresql-client \
     && rm -rf /var/lib/apt/lists/*
 RUN curl -sSL https://install.python-poetry.org | python -
 ENV PATH="/root/.local/bin:$PATH"

bin/import_areas_seed.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [ "$ENVIRONMENT" != "production" ]; then
+    unzip -o data/areas.psql.zip -d data
+    PGPASSWORD=password psql -U postgres -h db test_local-intelligence < data/areas.psql
+else
+    echo "This command cannot run in production environments."
+fi

hub/admin.py

@@ -6,6 +6,8 @@
     AreaData,
     DataSet,
     DataType,
+    ExternalDataSource,
+    GenericData,
     Membership,
     Organisation,
     Person,
@@ -262,3 +264,28 @@ class MembershipAdmin(admin.ModelAdmin):
     inline = [
         OrganisationInline,
     ]
+
+
+# External data source
+@admin.register(ExternalDataSource)
+class ExternalDataSourceAdmin(admin.ModelAdmin):
+    list_display = ("name", "orgname")
+
+    search_fields = ("name", "orgname")
+
+    def orgname(self, obj):
+        return obj.organisation.name
+
+    orgname.admin_order_field = "author"  # Allows column order sorting
+    orgname.short_description = "Author Name"  # Renames column head
+
+
+# Generic data
+@admin.register(GenericData)
+class GenericDataAdmin(admin.ModelAdmin):
+    list_display = ("name", "source", "value")
+
+    search_fields = ("name", "source", "value")
+
+    def source(self, obj):
+        return obj.data_type.data_set.external_data_source.name

hub/analytics.py

@@ -24,6 +24,22 @@ class AreaStat(TypedDict):
         gss: Optional[str]
         external_data: dict
 
+    def imported_data_count_located(self) -> int:
+        return (
+            self.get_analytics_queryset().filter(postcode_data__isnull=False).count()
+            or 0
+        )
+
+    def imported_data_count_unlocated(self) -> int:
+        return self.get_analytics_queryset().filter(postcode_data=None).count() or 0
+
+    def imported_data_geocoding_rate(self) -> float:
+        located = self.imported_data_count_located()
+        total = self.imported_data_count()
+        if total == 0:
+            return 0
+        return (located / total) * 100
+
     # TODO: Rename to e.g. row_count_by_political_boundary
     def imported_data_count_by_area(
         self,