diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 764c95f00..38a8c0dc2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest environment: testing services: - postgres: + db: image: kartoza/postgis:13 env: POSTGRES_USER: postgres @@ -31,7 +31,7 @@ jobs: # https://stackoverflow.com/questions/78593700/langchain-community-langchain-packages-giving-error-missing-1-required-keywor image: python:3.12.3 env: - DATABASE_URL: postgis://postgres:password@postgres:5432/local-intelligence + DATABASE_URL: postgis://postgres:password@db:5432/local-intelligence CACHE_FILE: /tmp/meep POETRY_VIRTUALENVS_CREATE: "false" GOOGLE_MAPS_API_KEY: ${{ secrets.GOOGLE_MAPS_API_KEY }} @@ -63,9 +63,11 @@ jobs: run: | curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null echo "deb https://ngrok-agent.s3.amazonaws.com buster main" | tee /etc/apt/sources.list.d/ngrok.list - apt-get update && apt-get install -y binutils gdal-bin libproj-dev ngrok less + apt-get update && apt-get install -y binutils gdal-bin libproj-dev ngrok less postgresql-client - name: Install poetry - run: curl -sSL https://install.python-poetry.org | python3 - + run: | + curl -sSL https://install.python-poetry.org | python3 - + ~/.local/bin/poetry self add poetry-plugin-export - name: Install python dependencies run: ~/.local/bin/poetry export --with dev --without-hashes -f requirements.txt --output requirements.txt && pip install -r requirements.txt - name: Start ngrok tunnelling @@ -86,6 +88,10 @@ jobs: run: gunicorn local_intelligence_hub.asgi:application -k uvicorn.workers.UvicornWorker --bind 0.0.0.0:8000 > server.log 2>&1 & - name: Run django tests run: cat .env && coverage run --source=. --branch manage.py test || (cat server.log && exit 1) + - name: Run geocoding tests in isolation + run: | + echo "RUN_GEOCODING_TESTS=1" >> .env + cat .env && python manage.py test hub.tests.test_external_data_source_parsers || (cat server.log && exit 1) - name: Generate coverage xml run: coverage xml - name: Upload coverage.xml diff --git a/.gitignore b/.gitignore index af1551a58..a330ec67b 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ __pycache__ .coverage .media data/**/* +!data/areas.psql.zip !data/.gitkeep !data/areas.psql.zip .next diff --git a/Dockerfile b/Dockerfile index 051e07c22..560d6d223 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ ENV INSIDE_DOCKER=1 \ SHELL=/bin/bash RUN curl -sL https://deb.nodesource.com/setup_22.x | bash RUN apt-get update && apt-get install -y \ - binutils gdal-bin libproj-dev git nodejs python3-dev \ + binutils gdal-bin libproj-dev git nodejs python3-dev postgresql-client \ && rm -rf /var/lib/apt/lists/* RUN curl -sSL https://install.python-poetry.org | python - ENV PATH="/root/.local/bin:$PATH" diff --git a/bin/import_areas_seed.sh b/bin/import_areas_seed.sh new file mode 100755 index 000000000..e4497c42c --- /dev/null +++ b/bin/import_areas_seed.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +if [ "$ENVIRONMENT" != "production" ]; then + unzip -o data/areas.psql.zip -d data + PGPASSWORD=password psql -U postgres -h db test_local-intelligence < data/areas.psql +else + echo "This command cannot run in production environments." +fi diff --git a/data/areas.psql.zip b/data/areas.psql.zip index 11f185841..5e69af571 100644 Binary files a/data/areas.psql.zip and b/data/areas.psql.zip differ diff --git a/hub/admin.py b/hub/admin.py index 15bb7166e..d007d3eb2 100644 --- a/hub/admin.py +++ b/hub/admin.py @@ -6,6 +6,8 @@ AreaData, DataSet, DataType, + ExternalDataSource, + GenericData, Membership, Organisation, Person, @@ -262,3 +264,28 @@ class MembershipAdmin(admin.ModelAdmin): inline = [ OrganisationInline, ] + + +# External data source +@admin.register(ExternalDataSource) +class ExternalDataSourceAdmin(admin.ModelAdmin): + list_display = ("name", "orgname") + + search_fields = ("name", "orgname") + + def orgname(self, obj): + return obj.organisation.name + + orgname.admin_order_field = "author" # Allows column order sorting + orgname.short_description = "Author Name" # Renames column head + + +# Generic data +@admin.register(GenericData) +class GenericDataAdmin(admin.ModelAdmin): + list_display = ("name", "source", "value") + + search_fields = ("name", "source", "value") + + def source(self, obj): + return obj.data_type.data_set.external_data_source.name diff --git a/hub/analytics.py b/hub/analytics.py index c41740f9a..d038a0527 100644 --- a/hub/analytics.py +++ b/hub/analytics.py @@ -24,6 +24,22 @@ class AreaStat(TypedDict): gss: Optional[str] external_data: dict + def imported_data_count_located(self) -> int: + return ( + self.get_analytics_queryset().filter(postcode_data__isnull=False).count() + or 0 + ) + + def imported_data_count_unlocated(self) -> int: + return self.get_analytics_queryset().filter(postcode_data=None).count() or 0 + + def imported_data_geocoding_rate(self) -> float: + located = self.imported_data_count_located() + total = self.imported_data_count() + if total == 0: + return 0 + return (located / total) * 100 + # TODO: Rename to e.g. row_count_by_political_boundary def imported_data_count_by_area( self, diff --git a/hub/data_imports/__init__.py b/hub/data_imports/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/hub/data_imports/geocoding_config.py b/hub/data_imports/geocoding_config.py new file mode 100644 index 000000000..e8ffa21b1 --- /dev/null +++ b/hub/data_imports/geocoding_config.py @@ -0,0 +1,640 @@ +import logging +import re +from enum import Enum +from typing import TYPE_CHECKING + +from django.conf import settings +from django.contrib.gis.geos import Point +from django.db.models import Q + +from asgiref.sync import sync_to_async + +from hub.data_imports.utils import get_update_data +from utils import google_maps, mapit_types +from utils.findthatpostcode import ( + get_example_postcode_from_area_gss, + get_postcode_from_coords_ftp, +) +from utils.postcodesIO import PostcodesIOResult +from utils.py import are_dicts_equal, ensure_list, find + +logger = logging.getLogger(__name__) +if TYPE_CHECKING: + from hub.models import DataType, ExternalDataSource, Loaders + + +def find_config_item(source: "ExternalDataSource", key: str, value, default=None): + return find( + source.geocoding_config.get("components", []), + lambda item: item.get(key, None) == value, + default, + ) + + +# enum of geocoders: postcodes_io, mapbox, google +class Geocoder(Enum): + POSTCODES_IO = "postcodes_io" + FINDTHATPOSTCODE = "findthatpostcode" + MAPBOX = "mapbox" + GOOGLE = "google" + AREA_GEOCODER_V2 = "AREA_GEOCODER_V2" + ADDRESS_GEOCODER_V2 = "ADDRESS_GEOCODER_V2" + COORDINATE_GEOCODER_V1 = "COORDINATE_GEOCODER_V1" + + +LATEST_AREA_GEOCODER = Geocoder.AREA_GEOCODER_V2 +LATEST_ADDRESS_GEOCODER = Geocoder.ADDRESS_GEOCODER_V2 +LATEST_COORDINATE_GEOCODER = Geocoder.COORDINATE_GEOCODER_V1 + + +def get_config_item_value( + source: "ExternalDataSource", config_item, record, default=None +): + if config_item is None: + return default + if config_item.get("field", None): + # Data comes from the member record + field = config_item.get("field", None) + value = source.get_record_field(record, field) + elif config_item.get("value", None): + # Data has been manually defined by the organiser + # (e.g. "all these venues are in Glasgow") + value = config_item.get("value", None) + return value or default + + +def get_config_item_field_value( + source: "ExternalDataSource", config_item, record, default=None +): + if config_item is None: + return default + if config_item.get("field", None): + # Data comes from the member record + field = config_item.get("field", None) + value = source.get_record_field(record, field) + return value or default + + +async def import_record( + record, + source: "ExternalDataSource", + data_type: "DataType", + loaders: "Loaders", +): + from hub.models import ExternalDataSource, GenericData + + id = source.get_record_id(record) + update_data = get_update_data(source, record) + update_data["geocode_data"] = update_data.get("geocode_data", {}) + update_data["geocode_data"]["config"] = source.geocoding_config + + # Try to identify the appropriate geocoder + geocoder: Geocoder = None + geocoding_config_type = source.geocoding_config.get("type", None) + importer_fn = None + if geocoding_config_type == ExternalDataSource.GeographyTypes.AREA: + geocoder = LATEST_AREA_GEOCODER + importer_fn = import_area_data + elif geocoding_config_type == ExternalDataSource.GeographyTypes.ADDRESS: + geocoder = LATEST_ADDRESS_GEOCODER + importer_fn = import_address_data + elif geocoding_config_type == ExternalDataSource.GeographyTypes.COORDINATES: + geocoder = LATEST_COORDINATE_GEOCODER + importer_fn = import_coordinate_data + else: + logger.debug(source.geocoding_config) + raise ValueError("geocoding_config is not a valid type") + + # check if geocoding_config and dependent fields are the same; if so, skip geocoding + try: + generic_data = await GenericData.objects.aget(data_type=data_type, data=id) + + # First check if the configs are the same + if ( + generic_data is not None + and + # Already is geocoded + generic_data.postcode_data is not None + and + # Has all the required geocoding metadata for us to check for deduplication + generic_data.geocode_data is not None + and generic_data.geocode_data.get("config", None) is not None + and are_dicts_equal( + generic_data.geocode_data["config"], source.geocoding_config + ) + # Add geocoding code versions are the same + and generic_data.geocoder == geocoder.value + ): + # Then, if so, check if the data has changed + geocoding_field_values = set() + search_terms_from_last_time = set() + for item in source.geocoding_config.get("components", []): + # new data + new_value = get_config_item_value(source, item, record) + geocoding_field_values.add(new_value) + # old data + old_value = get_config_item_value(source, item, generic_data.json) + search_terms_from_last_time.add(old_value) + # logger.debug("Equality check", id, item, new_value, old_value) + is_equal = geocoding_field_values == search_terms_from_last_time + + if is_equal: + # Don't bother with geocoding again. + # Mark this as such + # And simply update the other data. + update_data["geocode_data"] = generic_data.geocode_data or {} + update_data["geocode_data"]["skipped"] = True + return await GenericData.objects.aupdate_or_create( + data_type=data_type, data=id, defaults=update_data + ) + except GenericData.DoesNotExist: + # logger.debug("Generic Data doesn't exist, no equality check to be done", id) + pass + + update_data["geocode_data"]["skipped"] = False + + return await importer_fn( + record=record, + source=source, + data_type=data_type, + loaders=loaders, + update_data=update_data, + ) + + +async def import_area_data( + record, + source: "ExternalDataSource", + data_type: "DataType", + loaders: "Loaders", + update_data: dict, +): + from hub.models import Area, GenericData + + update_data["geocoder"] = LATEST_AREA_GEOCODER.value + + # Filter down geographies by the config + parent_area = None + area = None + geocoding_data = {} + steps = [] + + for item in source.geocoding_config.get("components", []): + parent_area = area + literal_lih_area_type__code = item.get("metadata", {}).get( + "lih_area_type__code", None + ) + literal_mapit_type = item.get("metadata", {}).get("mapit_type", None) + area_types = literal_lih_area_type__code or literal_mapit_type + literal_area_field = item.get("field", None) + raw_area_value = str(source.get_record_field(record, literal_area_field)) + + if area_types is None or literal_area_field is None or raw_area_value is None: + continue + + # make searchable for the MapIt database + lower_name = str(raw_area_value).lower() + + # E.g. ""Bristol, city of" becomes "bristol city" (https://mapit.mysociety.org/area/2561.html) + searchable_name = ( + re.sub(r"(.+), (.+) of", r"\1 \2", lower_name, flags=re.IGNORECASE) + or lower_name + ) + + # Sometimes MapIt uses "X council", sometimes "X city council" + # so try both, knowing we're already specifying the area type so it is safe. + searchable_name_sans_title = ( + re.sub(r"(.+), (.+) of", r"\1", lower_name, flags=re.IGNORECASE) + or lower_name + ) + + parsed_area_types = [str(s).upper() for s in ensure_list(area_types)] + + maybe_council = ( + # Check if using LIH area types + literal_lih_area_type__code is not None + and any([t in mapit_types.LIH_COUNCIL_TYPES for t in parsed_area_types]) + ) or ( + # Check if using MapIt types + literal_mapit_type is not None + and any([t in mapit_types.MAPIT_COUNCIL_TYPES for t in parsed_area_types]) + ) + + qs = Area.objects.select_related("area_type") + + if literal_lih_area_type__code is not None: + qs = qs.filter(area_type__code__in=parsed_area_types) + elif literal_mapit_type is not None: + qs = qs.filter(mapit_type__in=parsed_area_types) + + search_values = [ + raw_area_value, + lower_name, + searchable_name, + searchable_name_sans_title, + ] + + suffixes = [ + # try the values on their own + "" + ] + + if maybe_council: + # Mapit stores councils with their type in the name + # e.g. https://mapit.mysociety.org/area/2641.html + suffixes += [ + # add council suffixes + " council", + " city council", + " borough council", + " district council", + " county council", + ] + + # always try a code + or_statement_area_text_matcher = Q(gss__iexact=raw_area_value) + + # matrix of strings and suffixes + for value in list(set(search_values)): + for suffix in suffixes: + computed_value = f"{value}{suffix}" + # logger.debug("Trying ", computed_value) + # we also try trigram because different versions of the same name are used by organisers and researchers + or_statement_area_text_matcher |= Q( + name__unaccent__iexact=computed_value + ) + or_statement_area_text_matcher |= Q( + name__unaccent__trigram_similar=computed_value + ) + + qs = ( + qs.filter(or_statement_area_text_matcher) + .extra( + select={"exact_gss_match": "hub_area.gss = %s"}, + select_params=[raw_area_value.upper()], + ) + .extra( + # We round the similarity score to 1dp so that + # two areas with score 0.8 can be distinguished by which has a larger mapit_generation_high + # which wouldn't be possible if the invalid one had 0.82 and the valid one had 0.80 (because it was renamed "&" to "and") + select={ + "name_distance": "round(similarity(hub_area.name, %s)::numeric, 1)" + }, + select_params=[searchable_name_sans_title], + ) + .order_by( + # Prefer exact matches on GSS codes + "-exact_gss_match", + # Then prefer name similarity + "-name_distance", + # If the names are the same, prefer the most recent one + "-mapit_generation_high", + ) + ) + + # for debugging, but without all the polygon characters which spam up the terminal/logger + non_polygon_query = qs + + if parent_area is not None and parent_area.polygon is not None: + qs = qs.filter(polygon__intersects=parent_area.polygon) + + area = await qs.afirst() + + step = { + "type": "sql_area_matching", + "area_types": parsed_area_types, + "result": "failed" if area is None else "success", + "search_term": raw_area_value, + "data": ( + { + "centroid": area.polygon.centroid.json, + "name": area.name, + "id": area.id, + "gss": area.gss, + } + if area is not None + else None + ), + } + if settings.DEBUG: + step.update( + { + "query": str(non_polygon_query.query), + "parent_polygon_query": ( + parent_area.polygon.json + if parent_area is not None and parent_area.polygon is not None + else None + ), + } + ) + steps.append(step) + + if area is None: + break + else: + geocoding_data["area_fields"] = geocoding_data.get("area_fields", {}) + geocoding_data["area_fields"][area.area_type.code] = area.gss + update_data["geocode_data"].update({"data": geocoding_data}) + if area is not None: + sample_point = area.polygon.centroid + + # get postcodeIO result for area.coordinates + try: + postcode_data: PostcodesIOResult = await loaders[ + "postcodesIOFromPoint" + ].load(sample_point) + except Exception as e: + logger.error(f"Failed to get postcode data for {sample_point}: {e}") + postcode_data = None + + steps.append( + { + "task": "postcode_from_area_coordinates", + "service": Geocoder.POSTCODES_IO.value, + "result": "failed" if postcode_data is None else "success", + } + ) + + # Try a few other backup strategies (example postcode, another geocoder) + # to get postcodes.io data + if postcode_data is None: + postcode = await get_example_postcode_from_area_gss(area.gss) + steps.append( + { + "task": "postcode_from_area", + "service": Geocoder.FINDTHATPOSTCODE.value, + "result": "failed" if postcode is None else "success", + } + ) + if postcode is not None: + postcode_data = await loaders["postcodesIO"].load(postcode) + steps.append( + { + "task": "data_from_postcode", + "service": Geocoder.POSTCODES_IO.value, + "result": ("failed" if postcode_data is None else "success"), + } + ) + if postcode_data is None: + postcode = await get_postcode_from_coords_ftp(sample_point) + steps.append( + { + "task": "postcode_from_area_coordinates", + "service": Geocoder.FINDTHATPOSTCODE.value, + "result": "failed" if postcode is None else "success", + } + ) + if postcode is not None: + postcode_data = await loaders["postcodesIO"].load(postcode) + steps.append( + { + "task": "data_from_postcode", + "service": Geocoder.POSTCODES_IO.value, + "result": ("failed" if postcode_data is None else "success"), + } + ) + + update_data["postcode_data"] = postcode_data + else: + # Reset geocoding data + update_data["postcode_data"] = None + + # Update the geocode data regardless, for debugging purposes + update_data["geocode_data"].update({"steps": steps}) + + await GenericData.objects.aupdate_or_create( + data_type=data_type, data=source.get_record_id(record), defaults=update_data + ) + + +async def import_address_data( + record, + source: "ExternalDataSource", + data_type: "DataType", + loaders: "Loaders", + update_data: dict, +): + """ + Converts a record fetched from the API into + a GenericData record in the MEEP db. + + Used to batch-import data. + """ + from hub.models import GenericData + + update_data["geocoder"] = LATEST_ADDRESS_GEOCODER.value + + point = None + address_data = None + postcode_data = None + steps = [] + + # place_name — could be as simple as "Glasgow City Chambers" + place_name_config = find_config_item(source, "type", "place_name") + place_name_value = get_config_item_value(source, place_name_config, record) + has_dynamic_place_name_value = place_name_value and place_name_config.get( + "field", None + ) + # Address — the place_name (i.e. line1, location name) might be so specific + # that it's the only thing the organisers add, so don't require it + address_item = find_config_item(source, "type", "street_address") + street_address_value = get_config_item_field_value(source, address_item, record) + + if ( + street_address_value + or + # In the case of a list of Barclays addresses, the organiser might have defined + # { "type": "place_name", "value": "Barclays" } in the geocoding_config + # so that the spreadsheet only needs to contain the address. + # So we check for a dynamic place_name_value here, so we're not just geocoding a bunch of + # queries that are just "Barclays", "Barclays", "Barclays"... + has_dynamic_place_name_value + ): + # area + area_name_config = find_config_item(source, "type", "area_name") + area_name_value = get_config_item_value(source, area_name_config, record) + # Countries + countries_config = find_config_item(source, "type", "countries") + countries_value = ensure_list( + get_config_item_value(source, countries_config, record, source.countries) + ) + # join them into a string using join and a comma + query = ", ".join( + [ + x + for x in [place_name_value, street_address_value, area_name_value] + if x is not None and x != "" + ] + ) + address_data = await sync_to_async(google_maps.geocode_address)( + google_maps.GeocodingQuery( + query=query, + country=countries_value, + ) + ) + + steps.append( + { + "task": "address_from_query", + "service": Geocoder.GOOGLE.value, + "result": "failed" if address_data is None else "success", + "search_term": query, + "country": countries_value, + } + ) + + if address_data is not None: + update_data["geocode_data"]["data"] = address_data + point = ( + Point( + x=address_data.geometry.location.lng, + y=address_data.geometry.location.lat, + ) + if ( + address_data is not None + and address_data.geometry is not None + and address_data.geometry.location is not None + ) + else None + ) + + postcode_data = None + + # Prefer to get the postcode from the address data + # rather than inferring it from coordinate which might be wrong / non-canonical + postcode_component = find( + address_data.address_components, + lambda x: "postal_code" in x.get("types", []), + ) + if postcode_component is not None: + postcode = postcode_component.get("long_name", None) + + if postcode: + postcode_data = await loaders["postcodesIO"].load(postcode) + + steps.append( + { + "task": "data_from_postcode", + "service": Geocoder.POSTCODES_IO.value, + "result": "failed" if postcode_data is None else "success", + } + ) + # Else if no postcode's found, use the coordinates + if postcode_data is None and point is not None: + # Capture this so we have standardised Postcodes IO data for all records + # (e.g. for analytical queries that aggregate on region) + # even if the address is not postcode-specific (e.g. "London"). + # this can be gleaned from geocode_data__types, e.g. [ "administrative_area_level_1", "political" ] + postcode_data: PostcodesIOResult = await loaders[ + "postcodesIOFromPoint" + ].load(point) + + steps.append( + { + "task": "postcode_from_coordinates", + "service": Geocoder.POSTCODES_IO.value, + "result": "failed" if postcode_data is None else "success", + } + ) + + # Try a backup geocoder in case that one fails + if postcode_data is None: + postcode = await get_postcode_from_coords_ftp(point) + steps.append( + { + "task": "postcode_from_coordinates", + "service": Geocoder.FINDTHATPOSTCODE.value, + "result": "failed" if postcode_data is None else "success", + } + ) + if postcode is not None: + postcode_data = await loaders["postcodesIO"].load(postcode) + steps.append( + { + "task": "data_from_postcode", + "service": Geocoder.POSTCODES_IO.value, + "result": ( + "failed" if postcode_data is None else "success" + ), + } + ) + + update_data["geocode_data"].update({"steps": steps}) + update_data["postcode_data"] = postcode_data + update_data["point"] = point + + await GenericData.objects.aupdate_or_create( + data_type=data_type, data=source.get_record_id(record), defaults=update_data + ) + + +async def import_coordinate_data( + record, + source: "ExternalDataSource", + data_type: "DataType", + loaders: "Loaders", + update_data: dict, +): + from hub.models import GenericData + + update_data["geocoder"] = LATEST_COORDINATE_GEOCODER.value + + steps = [] + + raw_lng = get_config_item_value( + source, find_config_item(source, "type", "longitude"), record + ) + raw_lat = get_config_item_value( + source, find_config_item(source, "type", "latitude"), record + ) + postcode_data = None + point = None + + if raw_lng is not None and raw_lat is not None: + try: + point = Point( + x=float(raw_lng), + y=float(raw_lat), + ) + postcode_data = await loaders["postcodesIOFromPoint"].load(point) + + steps.append( + { + "task": "postcode_from_coordinates", + "service": Geocoder.POSTCODES_IO.value, + "result": "failed" if postcode_data is None else "success", + } + ) + except ValueError: + # If the coordinates are invalid, let it go. + pass + + # Try a backup geocoder in case that one fails + if point is not None and postcode_data is None: + postcode = await get_postcode_from_coords_ftp(point) + steps.append( + { + "task": "postcode_from_coordinates", + "service": Geocoder.FINDTHATPOSTCODE.value, + "result": "failed" if postcode_data is None else "success", + } + ) + if postcode is not None: + postcode_data = await loaders["postcodesIO"].load(postcode) + steps.append( + { + "task": "data_from_postcode", + "service": Geocoder.POSTCODES_IO.value, + "result": "failed" if postcode_data is None else "success", + } + ) + + update_data["geocode_data"].update({"steps": steps}) + update_data["postcode_data"] = postcode_data + update_data["point"] = point + + await GenericData.objects.aupdate_or_create( + data_type=data_type, + data=source.get_record_id(record), + defaults=update_data, + ) diff --git a/hub/data_imports/utils.py b/hub/data_imports/utils.py new file mode 100644 index 000000000..1a73aa88d --- /dev/null +++ b/hub/data_imports/utils.py @@ -0,0 +1,27 @@ +from datetime import datetime +from typing import TYPE_CHECKING + +from hub.validation import validate_and_format_phone_number +from utils.py import parse_datetime + +if TYPE_CHECKING: + from hub.models import ExternalDataSource + + +def get_update_data(source: "ExternalDataSource", record): + update_data = { + "json": source.get_record_dict(record), + } + + for field in source.import_fields: + if getattr(source, field, None) is not None: + value = source.get_record_field(record, getattr(source, field), field) + if field.endswith("_time_field"): + value: datetime = parse_datetime(value) + if field == "can_display_point_field": + value = bool(value) # cast None value to False + if field == "phone_field": + value = validate_and_format_phone_number(value, source.countries) + update_data[field.removesuffix("_field")] = value + + return update_data diff --git a/hub/graphql/mutations.py b/hub/graphql/mutations.py index 4f837f772..22f07dd83 100644 --- a/hub/graphql/mutations.py +++ b/hub/graphql/mutations.py @@ -10,6 +10,7 @@ import strawberry_django from asgiref.sync import async_to_sync from graphql import GraphQLError +from procrastinate.contrib.django.models import ProcrastinateJob from strawberry import auto from strawberry.field_extensions import InputMutationExtension from strawberry.types.info import Info @@ -20,6 +21,7 @@ from hub.graphql.types import model_types from hub.graphql.utils import graphql_type_to_dict from hub.models import BatchRequest +from hub.permissions import user_can_manage_source logger = logging.getLogger(__name__) @@ -253,6 +255,45 @@ async def import_all( return ExternalDataSourceAction(id=request_id, external_data_source=data_source) +@strawberry_django.mutation(extensions=[IsAuthenticated()]) +def cancel_import( + info: Info, external_data_source_id: str, request_id: str +) -> ExternalDataSourceAction: + data_source: models.ExternalDataSource = models.ExternalDataSource.objects.get( + id=external_data_source_id + ) + # Confirm user has access to this source + user = get_current_user(info) + assert user_can_manage_source(user, data_source) + # Update all remaining procrastinate jobs, cancel them + ProcrastinateJob.objects.filter( + args__external_data_source_id=external_data_source_id, + status__in=["todo", "doing"], + args__request_id=request_id, + ).update(status="cancelled") + BatchRequest.objects.filter(id=request_id).update(status="cancelled") + # + return ExternalDataSourceAction(id=request_id, external_data_source=data_source) + + +@strawberry_django.mutation(extensions=[IsAuthenticated()]) +def delete_all_records( + info: Info, external_data_source_id: str +) -> model_types.ExternalDataSource: + data_source = models.ExternalDataSource.objects.get(id=external_data_source_id) + # Confirm user has access to this source + user = get_current_user(info) + assert user_can_manage_source(user, data_source) + # Don't import more records, since we want to wipe 'em + ProcrastinateJob.objects.filter( + args__external_data_source_id=external_data_source_id, + status__in=["todo", "doing"], + ).update(status="cancelled") + # Delete all data + data_source.get_import_data().all().delete() + return models.ExternalDataSource.objects.get(id=external_data_source_id) + + @strawberry_django.input(models.ExternalDataSource, partial=True) class ExternalDataSourceInput: id: auto @@ -262,6 +303,7 @@ class ExternalDataSourceInput: organisation: auto geography_column: auto geography_column_type: auto + geocoding_config: Optional[strawberry.scalars.JSON] postcode_field: auto first_name_field: auto last_name_field: auto diff --git a/hub/graphql/schema.py b/hub/graphql/schema.py index 39cdb778d..a057bd586 100644 --- a/hub/graphql/schema.py +++ b/hub/graphql/schema.py @@ -181,6 +181,12 @@ class Mutation: ) import_all: mutation_types.ExternalDataSourceAction = mutation_types.import_all + cancel_import: mutation_types.ExternalDataSourceAction = ( + mutation_types.cancel_import + ) + delete_all_records: model_types.ExternalDataSource = ( + mutation_types.delete_all_records + ) create_map_report: model_types.MapReport = mutation_types.create_map_report update_map_report: model_types.MapReport = django_mutations.update( diff --git a/hub/graphql/types/model_types.py b/hub/graphql/types/model_types.py index be09f0b03..349e27353 100644 --- a/hub/graphql/types/model_types.py +++ b/hub/graphql/types/model_types.py @@ -49,6 +49,7 @@ class ProcrastinateJobStatus(Enum): doing = "doing" #: A worker is running the job succeeded = "succeeded" #: The job ended successfully failed = "failed" #: The job ended with an error + cancelled = "cancelled" #: The job was cancelled @strawberry_django.filters.filter( @@ -701,6 +702,9 @@ class AnalyticalAreaType(Enum): admin_district = "admin_district" admin_county = "admin_county" admin_ward = "admin_ward" + msoa = "msoa" + lsoa = "lsoa" + postcode = "postcode" european_electoral_region = "european_electoral_region" country = "country" @@ -745,7 +749,7 @@ class Analytics: def imported_data_count_by_area( self, analytical_area_type: AnalyticalAreaType, - layer_ids: Optional[List[str]], + layer_ids: Optional[List[str]] = [], ) -> List[GroupedDataCount]: data = self.imported_data_count_by_area( postcode_io_key=analytical_area_type.value, @@ -757,11 +761,35 @@ def imported_data_count_by_area( for datum in data ] + @strawberry_django.field + def imported_data_count_of_areas( + self, + analytical_area_type: AnalyticalAreaType, + layer_ids: Optional[List[str]] = [], + ) -> int: + data = self.imported_data_count_by_area( + postcode_io_key=analytical_area_type.value, + layer_ids=layer_ids, + ) + return max(len([d for d in data if d.get("count", 0) > 0]) or 0, 0) + + @strawberry_django.field + def imported_data_count_unlocated(self) -> int: + return self.imported_data_count_unlocated() + + @strawberry_django.field + def imported_data_count_located(self) -> int: + return self.imported_data_count_located() + + @strawberry_django.field + def imported_data_geocoding_rate(self) -> float: + return self.imported_data_geocoding_rate() + @strawberry_django.field def imported_data_by_area( self, analytical_area_type: AnalyticalAreaType, - layer_ids: Optional[List[str]], + layer_ids: Optional[List[str]] = [], ) -> List[GroupedData]: data = self.imported_data_by_area( postcode_io_key=analytical_area_type.value, @@ -900,6 +928,7 @@ class BaseDataSource(Analytics): last_update: auto geography_column: auto geography_column_type: auto + geocoding_config: JSON postcode_field: auto first_name_field: auto last_name_field: auto @@ -927,6 +956,10 @@ class BaseDataSource(Analytics): default_data_type: Optional[str] = attr_field() defaults: JSON = attr_field() + @strawberry_django.field + def uses_valid_geocoding_config(self) -> bool: + return self.uses_valid_geocoding_config() + @strawberry_django.field def is_import_scheduled(self: models.ExternalDataSource, info: Info) -> bool: job = self.get_scheduled_import_job() diff --git a/hub/management/commands/export_areas_as_sql.py b/hub/management/commands/export_areas_as_sql.py index 7385cac4b..a22c7633a 100644 --- a/hub/management/commands/export_areas_as_sql.py +++ b/hub/management/commands/export_areas_as_sql.py @@ -29,6 +29,7 @@ class TableConfig: output_column_templates={ "area_type_id": "(SELECT id FROM hub_areatype WHERE code = '{area_type_code}')" }, + exclude_columns=["mapit_all_names"], ), ] @@ -39,7 +40,15 @@ class Command(BaseCommand): without causing primary key conflicts. """ - def handle(self, *args, **options): + def add_arguments(self, parser): + parser.add_argument( + "-a", + "--all-names", + action="store_true", + help="Fetch alternative names from MapIt", + ) + + def handle(self, all_names: bool = False, *args, **options): print("Exporting areas and area types from current database to data/areas.psql") count = 0 output_file: Path = settings.BASE_DIR / "data" / "areas.psql" diff --git a/hub/management/commands/import_areas.py b/hub/management/commands/import_areas.py index 6b068727b..e9a0bfb75 100644 --- a/hub/management/commands/import_areas.py +++ b/hub/management/commands/import_areas.py @@ -1,4 +1,5 @@ import json +import logging from time import sleep # from django postgis @@ -8,43 +9,14 @@ from tqdm import tqdm from hub.models import Area, AreaType -from utils import mapit +from utils import mapit, mapit_types + +logger = logging.getLogger(__name__) class Command(BaseCommand): help = "Import basic area information from Mapit" - boundary_types = [ - { - "mapit_type": ["WMC"], - "name": "2023 Parliamentary Constituency", - "code": "WMC23", - "area_type": "Westminster Constituency", - "description": "Westminster Parliamentary Constituency boundaries, as created in 2023", - }, - { - "mapit_type": ["LBO", "UTA", "COI", "LGD", "CTY", "MTD"], - "name": "Single Tier Councils", - "code": "STC", - "area_type": "Single Tier Council", - "description": "Single Tier Council", - }, - { - "mapit_type": ["DIS", "NMD"], - "name": "District Councils", - "code": "DIS", - "area_type": "District Council", - "description": "District Council", - }, - { - "mapit_type": ["COI", "CPW", "DIW", "LBW", "LGW", "MTW", "UTE", "UTW"], - "name": "Wards", - "code": "WD23", - "area_type": "Electoral Ward", - "description": "Electoral wards", - }, - ] - def add_arguments(self, parser): parser.add_argument( "-q", "--quiet", action="store_true", help="Silence progress bars." @@ -58,8 +30,13 @@ def add_arguments(self, parser): def handle(self, quiet: bool = False, all_names: bool = False, *args, **options): self.mapit_client = mapit.MapIt() - for b_type in self.boundary_types: - areas = self.mapit_client.areas_of_type(b_type["mapit_type"]) + for b_type in mapit_types.boundary_types: + areas = self.mapit_client.areas_of_type( + b_type["mapit_type"], + { + "min_generation": 1, + }, + ) area_type, created = AreaType.objects.get_or_create( name=b_type["name"], code=b_type["code"], @@ -79,19 +56,34 @@ def handle(self, quiet: bool = False, all_names: bool = False, *args, **options) def import_area(self, area, area_type, all_names): area_details = self.mapit_client.area_details(area["id"]) if all_names else {} + + if "gss" not in area["codes"]: + # logger.debug(f"no gss code for {area['id']}") + return + + geom = None try: - geom = self.mapit_client.area_geometry(area["id"]) - geom = { - "type": "Feature", - "geometry": geom, - "properties": { - "PCON13CD": area["codes"]["gss"], - "name": area["name"], - "type": area_type.code, - "mapit_type": area["type"], - }, - } - geom_str = json.dumps(geom) + geom_already_loaded = Area.objects.filter( + gss=area["codes"]["gss"], polygon__isnull=False + ).exists() + if geom_already_loaded: + # Only fetch geometry data if required, to speed things up + # logger.debug(f"skipping geometry for {area['name']}") + pass + else: + geom = self.mapit_client.area_geometry(area["id"]) + + geom = { + "type": "Feature", + "geometry": geom, + "properties": { + "PCON13CD": area["codes"]["gss"], + "name": area["name"], + "type": area_type.code, + "mapit_type": area["type"], + }, + } + geom_str = json.dumps(geom) except mapit.NotFoundException: # pragma: no cover print(f"could not find mapit area for {area['name']}") geom = None @@ -102,8 +94,10 @@ def import_area(self, area, area_type, all_names): defaults={ "mapit_id": area["id"], "name": area["name"], - "mapit_type": area["type"], - "mapit_all_names": area_details.get("all_names"), + "mapit_type": area.get("type", None), + "mapit_generation_low": area.get("generation_low", None), + "mapit_generation_high": area.get("generation_high", None), + "mapit_all_names": area_details.get("all_names", None), }, ) diff --git a/hub/migrations/0130_rename_osm_data_genericdata_geocode_data_and_more.py b/hub/migrations/0130_rename_osm_data_genericdata_geocode_data_and_more.py index 68745284e..527ad0187 100644 --- a/hub/migrations/0130_rename_osm_data_genericdata_geocode_data_and_more.py +++ b/hub/migrations/0130_rename_osm_data_genericdata_geocode_data_and_more.py @@ -1,6 +1,8 @@ # Generated by Django 4.2.11 on 2024-06-10 20:00 from django.db import migrations, models +import hub.data_imports +import hub.data_imports.geocoding_config import hub.models @@ -31,7 +33,9 @@ class Migration(migrations.Migration): name="geocoder", field=models.CharField( blank=True, - default=hub.models.Geocoder["POSTCODES_IO"].value, + default=hub.data_imports.geocoding_config.Geocoder[ + "POSTCODES_IO" + ].value, max_length=1000, null=True, ), diff --git a/hub/migrations/0150_geocoding_config.py b/hub/migrations/0150_geocoding_config.py new file mode 100644 index 000000000..a648ec20d --- /dev/null +++ b/hub/migrations/0150_geocoding_config.py @@ -0,0 +1,24 @@ +# Generated by Django 4.2.11 on 2024-12-10 14:25 + +from django.db import migrations +from django.contrib.postgres.operations import TrigramExtension, UnaccentExtension +import django_jsonform.models.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ("hub", "0149_area_mapit_all_names"), + ] + + operations = [ + TrigramExtension(), + UnaccentExtension(), + migrations.AddField( + model_name="externaldatasource", + name="geocoding_config", + field=django_jsonform.models.fields.JSONField( + blank=True, default=list, null=True + ), + ), + ] diff --git a/hub/migrations/0151_area_mapit_generation_high_area_mapit_generation_low.py b/hub/migrations/0151_area_mapit_generation_high_area_mapit_generation_low.py new file mode 100644 index 000000000..34b09e304 --- /dev/null +++ b/hub/migrations/0151_area_mapit_generation_high_area_mapit_generation_low.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.11 on 2024-12-19 12:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("hub", "0150_geocoding_config"), + ] + + operations = [ + migrations.AddField( + model_name="area", + name="mapit_generation_high", + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name="area", + name="mapit_generation_low", + field=models.IntegerField(blank=True, null=True), + ), + ] diff --git a/hub/models.py b/hub/models.py index 1c7f91f6f..ef8149f07 100644 --- a/hub/models.py +++ b/hub/models.py @@ -5,7 +5,6 @@ import math import uuid from datetime import datetime, timedelta, timezone -from enum import Enum from typing import List, Optional, Self, Type, TypedDict, Union from urllib.parse import urlencode, urljoin @@ -60,6 +59,8 @@ import utils as lih_utils from hub.analytics import Analytics from hub.cache_keys import site_tile_filter_dict +from hub.data_imports import geocoding_config +from hub.data_imports.utils import get_update_data from hub.enrichment.sources import builtin_mapping_sources from hub.fields import EncryptedCharField, EncryptedTextField from hub.filters import Filter @@ -83,20 +84,13 @@ get_bulk_postcode_geo, get_bulk_postcode_geo_from_coords, ) -from utils.py import batched, ensure_list, get, is_maybe_id, parse_datetime +from utils.py import batched, ensure_list, get, is_maybe_id User = get_user_model() logger = get_simple_debug_logger(__name__) -# enum of geocoders: postcodes_io, mapbox, google -class Geocoder(Enum): - POSTCODES_IO = "postcodes_io" - MAPBOX = "mapbox" - GOOGLE = "google" - - class Organisation(models.Model): created_at = models.DateTimeField(auto_now_add=True) last_update = models.DateTimeField(auto_now=True) @@ -779,9 +773,7 @@ class GenericData(CommonData): public_url = models.URLField(max_length=2000, blank=True, null=True) social_url = models.URLField(max_length=2000, blank=True, null=True) geocode_data = JSONField(blank=True, null=True) - geocoder = models.CharField( - max_length=1000, blank=True, null=True, default=Geocoder.POSTCODES_IO.value - ) + geocoder = models.CharField(max_length=1000, blank=True, null=True) address = models.CharField(max_length=1000, blank=True, null=True) title = models.CharField(max_length=1000, blank=True, null=True) description = models.TextField(max_length=3000, blank=True, null=True) @@ -838,6 +830,8 @@ def save(self, *args, **kwargs): class Area(models.Model): mapit_id = models.CharField(max_length=30) mapit_type = models.CharField(max_length=30, db_index=True, blank=True, null=True) + mapit_generation_low = models.IntegerField(blank=True, null=True) + mapit_generation_high = models.IntegerField(blank=True, null=True) gss = models.CharField(max_length=30) name = models.CharField(max_length=200) mapit_all_names = models.JSONField(blank=True, null=True) @@ -1080,7 +1074,25 @@ class GeographyTypes(models.TextChoices): "PARLIAMENTARY_CONSTITUENCY_2024", "Constituency (2024)", ) - # TODO: LNG_LAT = "LNG_LAT", "Longitude and Latitude" + COORDINATES = ( + "COORDINATES", + "Coordinates", + ) + AREA = "AREA", "Area" + + geocoding_config = JSONField(blank=False, null=False, default=dict) + + def uses_valid_geocoding_config(self): + # TODO: Could replace this with a Pydantic schema or something + return ( + self.geocoding_config is not None + and isinstance(self.geocoding_config, dict) + and self.geocoding_config.get("type", None) is not None + and self.geocoding_config.get("type", None) in self.GeographyTypes.values + and self.geocoding_config.get("components", None) is not None + and isinstance(self.geocoding_config.get("components", None), list) + and len(self.geocoding_config.get("components", [])) > 0 + ) is True geography_column_type = TextChoicesField( choices_enum=GeographyTypes, @@ -1325,12 +1337,20 @@ def get_scheduled_batch_job_progress(self, parent_job: ProcrastinateJob, user=No jobs = self.event_log_queryset().filter(args__request_id=request_id).all() status = "todo" + number_of_jobs_ahead_in_queue = ( + ProcrastinateJob.objects.filter(id__lt=parent_job.id) + .filter(status__in=["todo", "doing"]) + .count() + ) + if any([job.status == "doing" for job in jobs]): status = "doing" elif any([job.status == "failed" for job in jobs]): status = "failed" elif all([job.status == "succeeded" for job in jobs]): status = "succeeded" + elif number_of_jobs_ahead_in_queue <= 0: + status = "succeeded" total = 0 statuses = dict() @@ -1349,12 +1369,6 @@ def get_scheduled_batch_job_progress(self, parent_job: ProcrastinateJob, user=No + statuses.get("doing", 0) ) - number_of_jobs_ahead_in_queue = ( - ProcrastinateJob.objects.filter(id__lt=parent_job.id) - .filter(status__in=["todo", "doing"]) - .count() - ) - time_started = ( ProcrastinateEvent.objects.filter(job_id=parent_job.id) .order_by("at") @@ -1584,6 +1598,8 @@ async def import_many(self, members: list): Copy data to this database for use in dashboarding features. """ + from hub.data_imports.geocoding_config import Geocoder + if not members: logger.error("import_many called with 0 records") return @@ -1611,29 +1627,19 @@ async def import_many(self, members: list): data_set=data_set, name=self.id, defaults={"data_type": "json"} ) - def get_update_data(record): - update_data = { - "json": self.get_record_dict(record), - } - - for field in self.import_fields: - if getattr(self, field, None) is not None: - value = self.get_record_field(record, getattr(self, field), field) - if field.endswith("_time_field"): - value: datetime = parse_datetime(value) - if field == "can_display_point_field": - value = bool(value) # cast None value to False - if field == "phone_field": - value = validate_and_format_phone_number(value, self.countries) - update_data[field.removesuffix("_field")] = value - - return update_data + loaders = await self.get_loaders() - if ( + if self.uses_valid_geocoding_config(): + await asyncio.gather( + *[ + geocoding_config.import_record(record, self, data_type, loaders) + for record in data + ] + ) + elif ( self.geography_column and self.geography_column_type == self.GeographyTypes.POSTCODE ): - loaders = await self.get_loaders() async def create_import_record(record): """ @@ -1642,13 +1648,14 @@ async def create_import_record(record): Used to batch-import data. """ - structured_data = get_update_data(record) + structured_data = get_update_data(self, record) postcode_data: PostcodesIOResult = await loaders["postcodesIO"].load( self.get_record_field(record, self.geography_column) ) update_data = { **structured_data, "postcode_data": postcode_data, + "geocoder": Geocoder.POSTCODES_IO.value, "point": ( Point( postcode_data["longitude"], @@ -1663,7 +1670,7 @@ async def create_import_record(record): ), } - await GenericData.objects.aupdate_or_create( + return await GenericData.objects.aupdate_or_create( data_type=data_type, data=self.get_record_id(record), defaults=update_data, @@ -1674,10 +1681,9 @@ async def create_import_record(record): self.geography_column and self.geography_column_type == self.GeographyTypes.WARD ): - loaders = await self.get_loaders() async def create_import_record(record): - structured_data = get_update_data(record) + structured_data = get_update_data(self, record) gss = self.get_record_field(record, self.geography_column) ward = await Area.objects.filter( area_type__code="WD23", @@ -1699,7 +1705,7 @@ async def create_import_record(record): "postcode_data": postcode_data, } - await GenericData.objects.aupdate_or_create( + return await GenericData.objects.aupdate_or_create( data_type=data_type, data=self.get_record_id(record), defaults=update_data, @@ -1712,7 +1718,6 @@ async def create_import_record(record): self.geography_column and self.geography_column_type == self.GeographyTypes.ADDRESS ): - loaders = await self.get_loaders() async def create_import_record(record): """ @@ -1721,7 +1726,7 @@ async def create_import_record(record): Used to batch-import data. """ - structured_data = get_update_data(record) + structured_data = get_update_data(self, record) address = self.get_record_field(record, self.geography_column) point = None address_data = None @@ -1771,7 +1776,7 @@ async def create_import_record(record): "point": point, } - await GenericData.objects.aupdate_or_create( + return await GenericData.objects.aupdate_or_create( data_type=data_type, data=self.get_record_id(record), defaults=update_data, @@ -1783,7 +1788,7 @@ async def create_import_record(record): # TODO: Re-implement this data as `AreaData`, linking each datum to an Area/AreaType as per `self.geography_column` and `self.geography_column_type`. # This will require importing other AreaTypes like admin_district, Ward for record in data: - update_data = get_update_data(record) + update_data = get_update_data(self, record) data, created = await GenericData.objects.aupdate_or_create( data_type=data_type, data=self.get_record_id(record), @@ -1836,7 +1841,7 @@ async def update_many(self, mapped_records: list[MappedMember], **kwargs): "Update many not implemented for this data source type." ) - def get_record_id(self, record): + def get_record_id(self, record) -> Optional[Union[str, int]]: """ Get the ID for a record. """ @@ -1868,7 +1873,6 @@ async def fetch_many_loader(self, keys): ] def get_import_data(self, **kwargs): - logger.debug(f"getting import data where external data source id is {self.id}") return GenericData.objects.filter( data_type__data_set__external_data_source_id=self.id ) @@ -1979,6 +1983,11 @@ async def get_loaders(self) -> Loaders: async def get_source_loaders(self) -> dict[str, Self]: # If this isn't preloaded, it is a sync function to use self.organisation org: Organisation = await sync_to_async(getattr)(self, "organisation") + loaders = {} + + if org is None: + return loaders + sources = ( org.get_external_data_sources( # Allow enrichment via sources shared with this data source's organisation @@ -1995,7 +2004,6 @@ async def get_source_loaders(self) -> dict[str, Self]: .all() ) - loaders = {} async for source in sources: loaders[str(source.id)] = source.data_loader_factory() @@ -2685,7 +2693,7 @@ def field_definitions(self): ] def get_record_id(self, record: dict): - return record[self.id_field] + return record.get(self.id_field, None) async def fetch_one(self, member_id): return self.df[self.df[self.id_field] == member_id].to_dict(orient="records")[0] @@ -2827,7 +2835,7 @@ async def fetch_all(self): return itertools.chain.from_iterable(self.table.iterate()) def get_record_id(self, record): - return record["id"] + return record.get("id", None) def get_record_field(self, record, field, field_type=None): record_dict = record["fields"] if "fields" in record else record @@ -3117,7 +3125,7 @@ def healthcheck(self): return False def get_record_id(self, record): - return record["id"] + return record.get("id", None) def get_record_field(self, record, field: str, field_type=None): field_options = [ @@ -3677,7 +3685,6 @@ def create_many(self, records): return created_records def get_import_data(self): - logger.debug(f"getting import data where action network source id is {self.id}") return GenericData.objects.filter( models.Q(data_type__data_set__external_data_source_id=self.id) & ( @@ -3943,7 +3950,7 @@ def create_sheet(self, sheet_name: str): del self.spreadsheet def get_record_id(self, record: dict): - return record[self.id_field] + return record.get(self.id_field, None) def get_record_dict(self, record: dict) -> dict: return record diff --git a/hub/permissions.py b/hub/permissions.py new file mode 100644 index 000000000..64a489f4f --- /dev/null +++ b/hub/permissions.py @@ -0,0 +1,2 @@ +def user_can_manage_source(user, source): + return source.organisation.members.filter(user=user).exists() diff --git a/hub/tests/fixtures/geocoding_cases.py b/hub/tests/fixtures/geocoding_cases.py new file mode 100644 index 000000000..b39a0fb44 --- /dev/null +++ b/hub/tests/fixtures/geocoding_cases.py @@ -0,0 +1,295 @@ +geocoding_cases = [ + # Name matching; cases that historically didn't work + { + "id": "1", + "council": "Barnsley", + "ward": "St Helens", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05000993", + }, + { + "id": "2", + "council": "North Lincolnshire", + "ward": "Brigg & Wolds", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05015081", + }, + { + "id": "3", + "council": "Test Valley", + "ward": "Andover Downlands", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05012085", + }, + { + "id": "4", + "council": "North Warwickshire", + "ward": "Baddesley and Grendon", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05007461", + }, + # Name rewriting required + { + "id": "5", + "council": "Herefordshire, County of", + "ward": "Credenhill", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05012957", + }, + # GSS code matching + { + "id": "999", + "council": "E08000016", + "ward": "E05000993", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05000993", + }, + # Misc + # Gwynedd Brithdir and Llanfachreth, Ganllwyd, Llanelltyd + # is Brithdir and Llanfachreth/Ganllwyd/Llanelltyd in MapIt + # https://mapit.mysociety.org/area/165898.html + { + "id": "6", + "council": "Gwynedd", + "ward": "Brithdir and Llanfachreth, Ganllwyd, Llanelltyd", + "expected_area_type_code": "WD23", + "expected_area_gss": "W05001514", + }, + # Isle of Anglesey Canolbarth Mon + # https://mapit.mysociety.org/area/144265.html + { + "id": "7", + "council": "Isle of Anglesey", + "ward": "Canolbarth Mon", + "expected_area_type_code": "WD23", + "expected_area_gss": "W05001496", + }, + # Denbighshire Rhyl T┼À Newydd + # Weird character in the name, probably needs trigram matching or something + # https://mapit.mysociety.org/area/166232.html + { + "id": "8", + "council": "Denbighshire", + "ward": "Rhyl T┼À Newydd", + "expected_area_type_code": "WD23", + "expected_area_gss": "W05001354", + }, + # Swansea B├┤n-y-maen + # Similarly, weird stuff in name + # Maybe it's a problem with the encoding? + # https://mapit.mysociety.org/area/165830.html — Bon-y-maen + { + "id": "9", + "council": "Swansea", + "ward": "B├┤n-y-maen", + "expected_area_type_code": "WD23", + "expected_area_gss": "W05001040", + }, + # Gwynedd Pendraw'r Llan + # Ought to be Pen draw Llyn + # https://mapit.mysociety.org/area/166296.html + { + "id": "10", + "council": "Gwynedd", + "ward": "Pendraw'r Llan", + "expected_area_type_code": "WD23", + "expected_area_gss": "W05001556", + }, + # Gwynedd Tre-garth a Mynydd Llandyg├íi + # https://mapit.mysociety.org/area/12219.html + # Tregarth & Mynydd Llandygai + { + "id": "542", + "council": "Gwynedd", + "ward": "Tre-garth a Mynydd Llandyg├íi", + "expected_area_type_code": "WD23", + "expected_area_gss": "W05001563", + }, + # A bunch of wards with the same name, should all point to different things + { + "id": "11", + "council": "Sandwell", + "ward": "Abbey", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05001260", + }, + { + "id": "12", + "council": "Nuneaton and Bedworth", + "ward": "Abbey", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05007474", + }, + { + "id": "13", + "council": "Redditch", + "ward": "Abbey", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05007868", + }, + { + "id": "14", + "council": "Shropshire", + "ward": "Abbey", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05008136", + }, + { + "id": "15", + "council": "Swale", + "ward": "Abbey", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05009544", + }, + { + "id": "16", + "council": "Leicester", + "ward": "Abbey", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05010458", + }, + { + "id": "17", + "council": "Cotswold", + "ward": "Abbey", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05010696", + }, + { + "id": "18", + "council": "Lincoln", + "ward": "Abbey", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05010784", + }, + { + "id": "19", + "council": "Cambridge", + "ward": "Abbey", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05013050", + }, + { + "id": "20", + "council": "Buckinghamshire", + "ward": "Abbey", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05013120", # old "E05002674", + }, + { + "id": "21", + "council": "Merton", + "ward": "Abbey", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05013810", + }, + { + "id": "22", + "council": "Reading", + "ward": "Abbey", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05013864", + # https://findthatpostcode.uk/areas/E05013864.html + }, + { + "id": "23", + "council": "Barking and Dagenham", + "ward": "Abbey", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05014053", + }, + { + "id": "24", + "council": "Rushcliffe", + "ward": "Abbey", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05014965", # old"E05009708", + }, + { + "id": "25", + "council": "Derby", + "ward": "Abbey", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05015507", + }, + { + "id": "26", + "council": "Dumfries and Galloway", + "ward": "Abbey", + "expected_area_type_code": "WD23", + "expected_area_gss": "S13002884", # old:"S13002537", + }, + # Nones + { + "id": "27", + "council": None, + "ward": None, + "expected_area_type_code": None, + "expected_area_gss": None, + }, + # + # More geocoding fails + { + "id": "28", + "council": "Wychavon", + "ward": "Bretforton & Offenham", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05015444", + }, + # East HertfordshireBuntingford + { + "id": "29", + "council": "East Hertfordshire", + "ward": "Buntingford", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05015362", + }, + # Neath Port TalbotCadoxton + { + "id": "30", + "council": "Neath Port Talbot", + "ward": "Cadoxton", + "expected_area_type_code": "WD23", + "expected_area_gss": "W05001689", + }, + # Great YarmouthCentral and Northgate + { + "id": "31", + "council": "Great Yarmouth", + "ward": "Central and Northgate", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05005788", + }, + # CarmarthenshirePontyberem + { + "id": "32", + "council": "Carmarthenshire", + "ward": "Pontyberem", + "expected_area_type_code": "WD23", + "expected_area_gss": "W05001219", + }, + # Geocoding based on historical parent areas + # that nonetheless point to live child areas + # + # RyedaleAmotherby & Ampleforth: + # This is a case where Ryedale is a now-defunct council + # but Amotherby & Ampleforth is still a live ward, under a new council. + # Because the dataset is ultimately about the ward, the geocoding should still work. + # Code-wise, this means looking for parent areas even when they're defunct + # if it means finding the right live child area. + { + "id": "33", + "council": "Ryedale", + "ward": "Amotherby & Ampleforth", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05014252", + }, + # Failed again + { + "id": "West LancashireBurscough Bridge & Rufford", + "ward": "Burscough Bridge & Rufford", + "council": "West Lancashire", + "expected_area_type_code": "WD23", + "expected_area_gss": "E05014930", + }, +] diff --git a/hub/tests/test_sources.py b/hub/tests/test_external_data_source_integrations.py similarity index 99% rename from hub/tests/test_sources.py rename to hub/tests/test_external_data_source_integrations.py index 93d0c0974..f5a866f4f 100644 --- a/hub/tests/test_sources.py +++ b/hub/tests/test_external_data_source_integrations.py @@ -375,7 +375,7 @@ async def test_refresh_many(self): [ models.ExternalDataSource.CUDRecord( postcode="E10 6EF", - email=f"gg{randint(0, 1000)}rardd@gmail.com", + email=f"hj{randint(0, 1000)}rardd@gmail.com", data=( { "addr1": "123 Colchester Rd", @@ -389,7 +389,7 @@ async def test_refresh_many(self): ), models.ExternalDataSource.CUDRecord( postcode="E5 0AA", - email=f"ag{randint(0, 1000)}rwefw@gmail.com", + email=f"kl{randint(0, 1000)}rwefw@gmail.com", data=( { "addr1": "Millfields Rd", diff --git a/hub/tests/test_external_data_source_parsers.py b/hub/tests/test_external_data_source_parsers.py new file mode 100644 index 000000000..d4d951beb --- /dev/null +++ b/hub/tests/test_external_data_source_parsers.py @@ -0,0 +1,585 @@ +import json +import os +import subprocess +from datetime import datetime, timezone +from unittest import skipIf + +from django.test import TestCase + +from asgiref.sync import async_to_sync + +from hub.models import Area, ExternalDataSource, LocalJSONSource +from hub.tests.fixtures.geocoding_cases import geocoding_cases +from hub.validation import validate_and_format_phone_number +from utils import mapit_types + +ignore_geocoding_tests = os.getenv("RUN_GEOCODING_TESTS") != "1" + + +class TestDateFieldParer(TestCase): + fixture = [ + { + "id": "1", + "date": "01/06/2024, 09:30", + "expected": datetime(2024, 6, 1, 9, 30, tzinfo=timezone.utc), + }, + { + "id": "2", + "date": "15/06/2024, 09:30", + "expected": datetime(2024, 6, 15, 9, 30, tzinfo=timezone.utc), + }, + { + "id": "3", + "date": "15/06/2024, 09:30:00", + "expected": datetime(2024, 6, 15, 9, 30, 0, tzinfo=timezone.utc), + }, + { + "id": "4", + "date": "2023-12-20 06:00:00", + "expected": datetime(2023, 12, 20, 6, 0, 0, tzinfo=timezone.utc), + }, + ] + + @classmethod + def setUpTestData(cls): + cls.source = LocalJSONSource.objects.create( + name="date_test", + id_field="id", + start_time_field="date", + data=[ + { + "id": d["id"], + "date": d["date"], + } + for d in cls.fixture + ], + ) + + # generate GenericData records + async_to_sync(cls.source.import_many)(cls.source.data) + + # test that the GenericData records have valid dates + cls.data = cls.source.get_import_data() + + def test_date_field(self): + for e in self.fixture: + d = self.data.get(data=e["id"]) + self.assertEqual(d.start_time, e["expected"]) + + +class TestPhoneFieldParser(TestCase): + fixture = [ + {"id": "bad1", "phone": "123456789", "expected": None}, + {"id": "good1", "phone": "07123456789", "expected": "+447123456789"}, + {"id": "good2", "phone": "+447123456789", "expected": "+447123456789"}, + ] + + @classmethod + def setUpTestData(cls): + cls.source = LocalJSONSource.objects.create( + name="phone_test", + id_field="id", + phone_field="phone", + countries=["GB"], + data=[ + { + "id": e["id"], + "phone": e["phone"], + } + for e in cls.fixture + ], + ) + + # generate GenericData records + async_to_sync(cls.source.import_many)(cls.source.data) + + # test that the GenericData records have valid, formatted phone field + cls.data = cls.source.get_import_data() + + def test_phone_field(self): + for e in self.fixture: + d = self.data.get(data=e["id"]) + self.assertEqual(d.phone, e["expected"]) + self.assertEqual(d.json["phone"], e["phone"]) + + def test_valid_phone_number_for_usa(self): + phone = "4155552671" + result = validate_and_format_phone_number(phone, ["US"]) + self.assertEqual(result, "+14155552671") + + +@skipIf(ignore_geocoding_tests, "It messes up data for other tests.") +class TestMultiLevelGeocoding(TestCase): + fixture = geocoding_cases + + @classmethod + def setUpTestData(cls): + subprocess.call("bin/import_areas_seed.sh") + + for d in cls.fixture: + if d["expected_area_gss"] is not None: + area = Area.objects.filter(gss=d["expected_area_gss"]).first() + if area is None: + print(f"Area not found, skipping: {d['expected_area_gss']}") + # remove the area from the test data so tests can run + index_of_data = next( + i for i, item in enumerate(cls.fixture) if item["id"] == d["id"] + ) + cls.fixture.pop(index_of_data) + + cls.source = LocalJSONSource.objects.create( + name="geo_test", + id_field="id", + data=cls.fixture.copy(), + geocoding_config={ + "type": ExternalDataSource.GeographyTypes.AREA, + "components": [ + { + "field": "council", + "metadata": {"lih_area_type__code": ["STC", "DIS"]}, + }, + {"field": "ward", "metadata": {"lih_area_type__code": "WD23"}}, + ], + }, + ) + + def test_geocoding_test_rig_is_valid(self): + self.assertGreaterEqual(Area.objects.count(), 19542) + self.assertGreaterEqual( + Area.objects.filter(polygon__isnull=False).count(), 19542 + ) + self.assertGreaterEqual(Area.objects.filter(area_type__code="DIS").count(), 164) + self.assertGreaterEqual(Area.objects.filter(area_type__code="STC").count(), 218) + self.assertGreaterEqual( + Area.objects.filter(area_type__code="WD23").count(), 8000 + ) + + # re-generate GenericData records + async_to_sync(self.source.import_many)(self.source.data) + + # load up the data for tests + self.data = self.source.get_import_data() + + for d in self.data: + try: + if d.json["expected_area_gss"] is not None: + area = Area.objects.get(gss=d.json["expected_area_gss"]) + self.assertIsNotNone(area) + except Area.DoesNotExist: + pass + + def test_geocoding_matches(self): + # re-generate GenericData records + async_to_sync(self.source.import_many)(self.source.data) + + # load up the data for tests + self.data = self.source.get_import_data() + + self.assertEqual( + len(self.data), + len(self.source.data), + "All data should be imported.", + ) + + for d in self.data: + try: + try: + try: + if d.json["ward"] is None: + self.assertIsNone( + d.postcode_data, "None shouldn't geocode." + ) + continue + elif d.json["expected_area_gss"] is None: + self.assertIsNone( + d.postcode_data, "Expect MapIt to have failed." + ) + continue + elif d.json["expected_area_gss"] is not None: + self.assertEqual( + d.geocode_data["data"]["area_fields"][ + d.json["expected_area_type_code"] + ], + d.json["expected_area_gss"], + ) + self.assertFalse( + d.geocode_data["skipped"], "Geocoding should be done." + ) + self.assertIsNotNone(d.postcode_data) + self.assertGreaterEqual( + len(d.geocode_data["steps"]), + 3, + "Geocoding outcomes should be debuggable, for future development.", + ) + except KeyError: + raise AssertionError("Expected geocoding data was missing.") + except AssertionError as e: + print(e) + print("Geocoding failed:", d.id, json.dumps(d.json, indent=4)) + print("--Geocode data:", d.id, json.dumps(d.geocode_data, indent=4)) + print( + "--Postcode data:", d.id, json.dumps(d.postcode_data, indent=4) + ) + raise + except TypeError as e: + print(e) + print("Geocoding failed:", d.id, json.dumps(d.json, indent=4)) + print("--Geocode data:", d.id, json.dumps(d.geocode_data, indent=4)) + print("--Postcode data:", d.id, json.dumps(d.postcode_data, indent=4)) + raise + + def test_by_mapit_types(self): + """ + Geocoding should work identically on more granular mapit_types + """ + + self.source.geocoding_config = { + "type": ExternalDataSource.GeographyTypes.AREA, + "components": [ + { + "field": "council", + "metadata": {"mapit_type": mapit_types.MAPIT_COUNCIL_TYPES}, + }, + { + "field": "ward", + "metadata": {"mapit_type": mapit_types.MAPIT_WARD_TYPES}, + }, + ], + } + self.source.save() + + # re-generate GenericData records + async_to_sync(self.source.import_many)(self.source.data) + + # load up the data for tests + self.data = self.source.get_import_data() + + for d in self.data: + try: + try: + if d.json["ward"] is None: + self.assertIsNone(d.postcode_data, "None shouldn't geocode.") + continue + elif d.json["expected_area_gss"] is None: + self.assertIsNone( + d.postcode_data, "Expect MapIt to have failed." + ) + continue + elif d.json["expected_area_gss"] is not None: + self.assertEqual( + d.geocode_data["data"]["area_fields"][ + d.json["expected_area_type_code"] + ], + d.json["expected_area_gss"], + ) + self.assertIsNotNone(d.postcode_data) + self.assertDictEqual( + dict(self.source.geocoding_config), + dict(d.geocode_data.get("config", {})), + "Geocoding config should be the same as the source's", + ) + self.assertFalse( + d.geocode_data["skipped"], "Geocoding should be done." + ) + except KeyError: + raise AssertionError("Expected geocoding data was missing.") + except AssertionError as e: + print(e) + print("Geocoding failed:", d.id, json.dumps(d.json, indent=4)) + print("--Geocode data:", d.id, json.dumps(d.geocode_data, indent=4)) + print("--Postcode data:", d.id, json.dumps(d.postcode_data, indent=4)) + raise + + def test_skipping(self): + """ + If all geocoding config is the same, and the data is the same too, then geocoding should be skipped + """ + # generate GenericData records — first time they should all geocode + async_to_sync(self.source.import_many)(self.source.data) + + # re-generate GenericData records — this time, they should all skip + async_to_sync(self.source.import_many)(self.source.data) + + # load up the data for tests + self.data = self.source.get_import_data() + + for d in self.data: + try: + try: + if d.json["expected_area_gss"] is not None: + self.assertTrue( + d.geocode_data["skipped"], "Geocoding should be skipped." + ) + self.assertIsNotNone(d.postcode_data) + except KeyError: + raise AssertionError("Expected geocoding data was missing.") + except AssertionError as e: + print(e) + print( + "Geocoding was repeated unecessarily:", + d.id, + json.dumps(d.json, indent=4), + ) + print("--Geocode data:", d.id, json.dumps(d.geocode_data, indent=4)) + print("--Postcode data:", d.id, json.dumps(d.postcode_data, indent=4)) + raise + + +@skipIf(ignore_geocoding_tests, "It messes up data for other tests.") +class TestComplexAddressGeocoding(TestCase): + @classmethod + def setUpTestData(cls): + cls.source = LocalJSONSource.objects.create( + name="address_test", + id_field="id", + data=[ + { + "id": "1", + "venue_name": "Boots", + "address": "415-417 Victoria Rd, Govanhill", + "expected_postcode": "G42 8RW", + }, + { + "id": "2", + "venue_name": "Lidl", + "address": "Victoria Road", + "expected_postcode": "G42 7RP", + }, + { + "id": "3", + "venue_name": "Sainsbury's", + "address": "Gordon Street", + "expected_postcode": "G1 3RS", + }, + { + # Special case: "online" + "id": "4", + "venue_name": "Barclays", + "address": "online", + "expected_postcode": None, + }, + # Edge cases + { + "id": "5", + "venue_name": None, + "address": "100 Torisdale Street", + "expected_postcode": "G42 8PH", + }, + { + "id": "6", + "venue_name": "Glasgow City Council Chambers", + "address": None, + "expected_postcode": "G2 5AF", + }, + { + "id": "7", + "venue_name": "Boots", + "address": None, + "expected_postcode": None, + }, + { + "id": "8", + "venue_name": None, + "address": None, + "expected_postcode": None, + }, + ], + geocoding_config={ + "type": ExternalDataSource.GeographyTypes.ADDRESS, + "components": [ + {"type": "place_name", "field": "venue_name"}, + {"type": "street_address", "field": "address"}, + {"type": "area", "value": "Glasgow"}, + ], + }, + ) + + def test_geocoding_matches(self): + # re-generate GenericData records + async_to_sync(self.source.import_many)(self.source.data) + + # load up the data for tests + self.data = self.source.get_import_data() + + self.assertEqual( + len(self.data), + len(self.source.data), + "All data should be imported.", + ) + + for d in self.data: + try: + try: + if d.json["expected_postcode"] is not None: + self.assertIsNotNone(d.postcode_data) + self.assertEqual( + d.postcode_data["postcode"], d.json["expected_postcode"] + ) + self.assertGreaterEqual(len(d.geocode_data["steps"]), 1) + except KeyError: + raise AssertionError("Expected geocoding data was missing.") + except AssertionError as e: + print(e) + print("Geocoding failed:", d.id, json.dumps(d.json, indent=4)) + print("--Geocode data:", d.id, json.dumps(d.geocode_data, indent=4)) + print("--Postcode data:", d.id, json.dumps(d.postcode_data, indent=4)) + raise + + def test_skipping(self): + """ + If all geocoding config is the same, and the data is the same too, then geocoding should be skipped + """ + # generate GenericData records — first time they should all geocode + async_to_sync(self.source.import_many)(self.source.data) + + # re-generate GenericData records — this time, they should all skip + async_to_sync(self.source.import_many)(self.source.data) + + # load up the data for tests + self.data = self.source.get_import_data() + + for d in self.data: + try: + try: + if d.json["expected_postcode"] is not None: + self.assertIsNotNone(d.postcode_data) + self.assertTrue( + d.geocode_data["skipped"], "Geocoding should be skipped." + ) + except KeyError: + raise AssertionError("Expected geocoding data was missing.") + except AssertionError as e: + print(e) + print( + "Geocoding was repeated unecessarily:", + d.id, + json.dumps(d.json, indent=4), + ) + print("--Geocode data:", d.id, json.dumps(d.geocode_data, indent=4)) + print("--Postcode data:", d.id, json.dumps(d.postcode_data, indent=4)) + raise + + +@skipIf(ignore_geocoding_tests, "It messes up data for other tests.") +class TestCoordinateGeocoding(TestCase): + @classmethod + def setUpTestData(cls): + cls.source = LocalJSONSource.objects.create( + name="coordinates_test", + id_field="id", + data=[ + { + "id": "1", + "longitude": -1.342881, + "latitude": 51.846073, + "expected_postcode": "OX20 1ND", + }, + { + # Should work with strings too + "id": "2", + "longitude": "-1.702695", + "latitude": "52.447681", + "expected_postcode": "B92 0HJ", + }, + { + "id": "3", + "longitude": " -1.301473", + "latitude": 53.362753, + "expected_postcode": "S26 2GA", + }, + { + # Handle failure cases gracefully + "id": "4", + "longitude": -4.2858, + "latitude": None, + "expected_postcode": None, + }, + # Gracefully handle non-numeric coordinates + { + "id": "5", + "longitude": "invalid", + "latitude": "invalid", + "expected_postcode": None, + }, + # Gracefully handle crazy big coordinates + { + "id": "6", + "longitude": 0, + "latitude": 1000, + "expected_postcode": None, + }, + ], + # Resulting address query should be something like "Barclays, Victoria Road, Glasgow" + geocoding_config={ + "type": ExternalDataSource.GeographyTypes.COORDINATES, + "components": [ + {"type": "latitude", "field": "latitude"}, + {"type": "longitude", "field": "longitude"}, + ], + }, + ) + + def test_geocoding_matches(self): + # re-generate GenericData records + async_to_sync(self.source.import_many)(self.source.data) + + # load up the data for tests + self.data = self.source.get_import_data() + + self.assertEqual( + len(self.data), + len(self.source.data), + "All data should be imported.", + ) + + for d in self.data: + try: + try: + if d.json["expected_postcode"] is not None: + self.assertIsNotNone(d.postcode_data) + self.assertEqual( + d.postcode_data["postcode"], d.json["expected_postcode"] + ) + self.assertGreaterEqual(len(d.geocode_data["steps"]), 1) + except KeyError: + raise AssertionError("Expected geocoding data was missing.") + except AssertionError as e: + print(e) + print("Geocoding failed:", d.id, json.dumps(d.json, indent=4)) + print("--Geocode data:", d.id, json.dumps(d.geocode_data, indent=4)) + print("--Postcode data:", d.id, json.dumps(d.postcode_data, indent=4)) + raise + + def test_skipping(self): + """ + If all geocoding config is the same, and the data is the same too, then geocoding should be skipped + """ + # generate GenericData records — first time they should all geocode + async_to_sync(self.source.import_many)(self.source.data) + + # re-generate GenericData records — this time, they should all skip + async_to_sync(self.source.import_many)(self.source.data) + + # load up the data for tests + self.data = self.source.get_import_data() + + for d in self.data: + try: + try: + if d.json["expected_postcode"] is not None: + self.assertIsNotNone(d.postcode_data) + self.assertTrue( + d.geocode_data["skipped"], "Geocoding should be skipped." + ) + except KeyError: + raise AssertionError("Expected geocoding data was missing.") + except AssertionError as e: + print(e) + print( + "Geocoding was repeated unecessarily:", + d.id, + json.dumps(d.json, indent=4), + ) + print("--Geocode data:", d.id, json.dumps(d.geocode_data, indent=4)) + print("--Postcode data:", d.id, json.dumps(d.postcode_data, indent=4)) + raise diff --git a/hub/tests/test_import_areas.py b/hub/tests/test_import_areas.py index 77cd07b1d..b4ad08424 100644 --- a/hub/tests/test_import_areas.py +++ b/hub/tests/test_import_areas.py @@ -7,7 +7,7 @@ from utils.mapit import MapIt -def mock_areas_of_type(types): +def mock_areas_of_type(types, *args, **kwargs): if "WMC" in types: return [ { diff --git a/hub/tests/test_source_parser.py b/hub/tests/test_source_parser.py deleted file mode 100644 index 1bc4533a8..000000000 --- a/hub/tests/test_source_parser.py +++ /dev/null @@ -1,94 +0,0 @@ -from datetime import datetime, timezone - -from django.test import TestCase - -from hub.models import LocalJSONSource -from hub.validation import validate_and_format_phone_number - - -class TestSourceParser(TestCase): - async def test_date_field(self): - fixture = [ - { - "id": "1", - "date": "01/06/2024, 09:30", - "expected": datetime(2024, 6, 1, 9, 30, tzinfo=timezone.utc), - }, - { - "id": "2", - "date": "15/06/2024, 09:30", - "expected": datetime(2024, 6, 15, 9, 30, tzinfo=timezone.utc), - }, - { - "id": "3", - "date": "15/06/2024, 09:30:00", - "expected": datetime(2024, 6, 15, 9, 30, 0, tzinfo=timezone.utc), - }, - { - "id": "4", - "date": "2023-12-20 06:00:00", - "expected": datetime(2023, 12, 20, 6, 0, 0, tzinfo=timezone.utc), - }, - ] - - source = await LocalJSONSource.objects.acreate( - name="date_test", - id_field="id", - start_time_field="date", - data=[ - { - "id": d["id"], - "date": d["date"], - } - for d in fixture - ], - ) - - # generate GenericData records - await source.import_many(source.data) - - # test that the GenericData records have valid dates - data = source.get_import_data() - - for e in fixture: - d = await data.aget(data=e["id"]) - self.assertEqual(d.start_time, e["expected"]) - - -class TestPhoneField(TestCase): - async def test_phone_field(self): - fixture = [ - {"id": "bad1", "phone": "123456789", "expected": None}, - {"id": "good1", "phone": "07123456789", "expected": "+447123456789"}, - {"id": "good2", "phone": "+447123456789", "expected": "+447123456789"}, - ] - - source = await LocalJSONSource.objects.acreate( - name="phone_test", - id_field="id", - phone_field="phone", - countries=["GB"], - data=[ - { - "id": e["id"], - "phone": e["phone"], - } - for e in fixture - ], - ) - - # generate GenericData records - await source.import_many(source.data) - - # test that the GenericData records have valid, formatted phone field - data = source.get_import_data() - - for e in fixture: - d = await data.aget(data=e["id"]) - self.assertEqual(d.phone, e["expected"]) - self.assertEqual(d.json["phone"], e["phone"]) - - def test_valid_phone_number_for_usa(self): - phone = "4155552671" - result = validate_and_format_phone_number(phone, ["US"]) - self.assertEqual(result, "+14155552671") diff --git a/local_intelligence_hub/settings.py b/local_intelligence_hub/settings.py index 0e0ab5180..47a34cf42 100644 --- a/local_intelligence_hub/settings.py +++ b/local_intelligence_hub/settings.py @@ -234,8 +234,10 @@ # Application definition INSTALLED_APPS = [ + "django.contrib.gis", "django.contrib.admin", "django.contrib.auth", + "django.contrib.postgres", "polymorphic", "django.contrib.contenttypes", "django.contrib.sessions", diff --git a/nextjs/.eslintrc.json b/nextjs/.eslintrc.json index b4af262bd..c07c39cdf 100644 --- a/nextjs/.eslintrc.json +++ b/nextjs/.eslintrc.json @@ -1,5 +1,8 @@ { "extends": ["next/core-web-vitals", "prettier"], - "rules": { "react/no-unescaped-entities": 0 }, + "rules": { + "react/no-unescaped-entities": 0, + "react/no-children-prop": 0 + }, "ignorePatterns": ["src/generated"] } diff --git a/nextjs/src/__generated__/gql.ts b/nextjs/src/__generated__/gql.ts index 8113882a6..c8e92855f 100644 --- a/nextjs/src/__generated__/gql.ts +++ b/nextjs/src/__generated__/gql.ts @@ -23,18 +23,20 @@ const documents = { "\n mutation PerformPasswordReset(\n $token: String!\n $password1: String!\n $password2: String!\n ) {\n performPasswordReset(\n token: $token\n newPassword1: $password1\n newPassword2: $password2\n ) {\n errors\n success\n }\n }\n": types.PerformPasswordResetDocument, "\n mutation ResetPassword($email: String!) {\n requestPasswordReset(email: $email) {\n errors\n success\n }\n }\n": types.ResetPasswordDocument, "\n query ListOrganisations($currentOrganisationId: ID!) {\n myOrganisations(filters: { id: $currentOrganisationId }) {\n id\n externalDataSources {\n id\n name\n dataType\n connectionDetails {\n ... on AirtableSource {\n baseId\n tableId\n }\n ... on MailchimpSource {\n apiKey\n listId\n }\n }\n crmType\n autoImportEnabled\n autoUpdateEnabled\n jobs(pagination: { limit: 10 }) {\n lastEventAt\n status\n }\n updateMapping {\n source\n sourcePath\n destinationColumn\n }\n sharingPermissions {\n id\n organisation {\n id\n name\n }\n }\n }\n sharingPermissionsFromOtherOrgs {\n id\n externalDataSource {\n id\n name\n dataType\n crmType\n organisation {\n name\n }\n }\n }\n }\n }\n": types.ListOrganisationsDocument, - "\n query GetSourceMapping($ID: ID!) {\n externalDataSource(pk: $ID) {\n id\n autoImportEnabled\n autoUpdateEnabled\n allowUpdates\n hasWebhooks\n updateMapping {\n destinationColumn\n source\n sourcePath\n }\n fieldDefinitions {\n label\n value\n description\n editable\n }\n crmType\n geographyColumn\n geographyColumnType\n postcodeField\n firstNameField\n lastNameField\n emailField\n phoneField\n addressField\n canDisplayPointField\n }\n }\n": types.GetSourceMappingDocument, - "\n query TestDataSource($input: CreateExternalDataSourceInput!) {\n testDataSource(input: $input) {\n __typename\n crmType\n fieldDefinitions {\n label\n value\n description\n editable\n }\n geographyColumn\n geographyColumnType\n healthcheck\n predefinedColumnNames\n defaultDataType\n remoteName\n allowUpdates\n defaults\n oauthCredentials\n }\n }\n": types.TestDataSourceDocument, + "\n query GetSourceMapping($ID: ID!) {\n externalDataSource(pk: $ID) {\n id\n autoImportEnabled\n autoUpdateEnabled\n allowUpdates\n hasWebhooks\n updateMapping {\n destinationColumn\n source\n sourcePath\n }\n fieldDefinitions {\n label\n value\n description\n editable\n }\n crmType\n geographyColumn\n geographyColumnType\n geocodingConfig\n usesValidGeocodingConfig\n postcodeField\n firstNameField\n lastNameField\n emailField\n phoneField\n addressField\n canDisplayPointField\n }\n }\n": types.GetSourceMappingDocument, + "\n query TestDataSource($input: CreateExternalDataSourceInput!) {\n testDataSource(input: $input) {\n __typename\n crmType\n fieldDefinitions {\n label\n value\n description\n editable\n }\n geographyColumn\n geographyColumnType\n geocodingConfig\n usesValidGeocodingConfig\n healthcheck\n predefinedColumnNames\n defaultDataType\n remoteName\n allowUpdates\n defaults\n oauthCredentials\n }\n }\n": types.TestDataSourceDocument, "\n query GoogleSheetsOauthUrl($redirectUrl: String!) {\n googleSheetsOauthUrl(redirectUrl: $redirectUrl)\n }\n": types.GoogleSheetsOauthUrlDocument, "\n query GoogleSheetsOauthCredentials($redirectSuccessUrl: String!) {\n googleSheetsOauthCredentials(redirectSuccessUrl: $redirectSuccessUrl)\n }\n": types.GoogleSheetsOauthCredentialsDocument, "\n mutation CreateSource($input: CreateExternalDataSourceInput!) {\n createExternalDataSource(input: $input) {\n code\n errors {\n message\n }\n result {\n id\n name\n crmType\n dataType\n allowUpdates\n }\n }\n }\n": types.CreateSourceDocument, - "\n query AutoUpdateCreationReview($ID: ID!) {\n externalDataSource(pk: $ID) {\n id\n name\n geographyColumn\n geographyColumnType\n dataType\n crmType\n autoImportEnabled\n autoUpdateEnabled\n updateMapping {\n source\n sourcePath\n destinationColumn\n }\n jobs(pagination: { limit: 10 }) {\n lastEventAt\n status\n }\n automatedWebhooks\n webhookUrl\n ...DataSourceCard\n }\n }\n \n": types.AutoUpdateCreationReviewDocument, - "\n query ExternalDataSourceInspectPage($ID: ID!) {\n externalDataSource(pk: $ID) {\n id\n name\n dataType\n remoteUrl\n crmType\n connectionDetails {\n ... on AirtableSource {\n apiKey\n baseId\n tableId\n }\n ... on MailchimpSource {\n apiKey\n listId\n }\n ... on ActionNetworkSource {\n apiKey\n groupSlug\n }\n ... on TicketTailorSource {\n apiKey\n }\n }\n lastImportJob {\n id\n lastEventAt\n status\n }\n lastUpdateJob {\n id\n lastEventAt\n status\n }\n autoImportEnabled\n autoUpdateEnabled\n hasWebhooks\n allowUpdates\n automatedWebhooks\n webhookUrl\n webhookHealthcheck\n geographyColumn\n geographyColumnType\n postcodeField\n firstNameField\n lastNameField\n fullNameField\n emailField\n phoneField\n addressField\n titleField\n descriptionField\n imageField\n startTimeField\n endTimeField\n publicUrlField\n socialUrlField\n canDisplayPointField\n isImportScheduled\n importProgress {\n id\n hasForecast\n status\n total\n succeeded\n estimatedFinishTime\n actualFinishTime\n inQueue\n numberOfJobsAheadInQueue\n sendEmail\n }\n isUpdateScheduled\n updateProgress {\n id\n hasForecast\n status\n total\n succeeded\n estimatedFinishTime\n actualFinishTime\n inQueue\n numberOfJobsAheadInQueue\n sendEmail\n }\n importedDataCount\n fieldDefinitions {\n label\n value\n description\n editable\n }\n updateMapping {\n source\n sourcePath\n destinationColumn\n }\n sharingPermissions {\n id\n }\n organisation {\n id\n name\n }\n }\n }\n": types.ExternalDataSourceInspectPageDocument, + "\n query AutoUpdateCreationReview($ID: ID!) {\n externalDataSource(pk: $ID) {\n id\n name\n geographyColumn\n geographyColumnType\n geocodingConfig\n usesValidGeocodingConfig\n dataType\n crmType\n autoImportEnabled\n autoUpdateEnabled\n updateMapping {\n source\n sourcePath\n destinationColumn\n }\n jobs(pagination: { limit: 10 }) {\n lastEventAt\n status\n }\n automatedWebhooks\n webhookUrl\n ...DataSourceCard\n }\n }\n \n": types.AutoUpdateCreationReviewDocument, + "\n query ExternalDataSourceInspectPage($ID: ID!) {\n externalDataSource(pk: $ID) {\n id\n name\n dataType\n remoteUrl\n crmType\n connectionDetails {\n ... on AirtableSource {\n apiKey\n baseId\n tableId\n }\n ... on MailchimpSource {\n apiKey\n listId\n }\n ... on ActionNetworkSource {\n apiKey\n groupSlug\n }\n ... on TicketTailorSource {\n apiKey\n }\n }\n lastImportJob {\n id\n lastEventAt\n status\n }\n lastUpdateJob {\n id\n lastEventAt\n status\n }\n autoImportEnabled\n autoUpdateEnabled\n hasWebhooks\n allowUpdates\n automatedWebhooks\n webhookUrl\n webhookHealthcheck\n geographyColumn\n geographyColumnType\n geocodingConfig\n usesValidGeocodingConfig\n postcodeField\n firstNameField\n lastNameField\n fullNameField\n emailField\n phoneField\n addressField\n titleField\n descriptionField\n imageField\n startTimeField\n endTimeField\n publicUrlField\n socialUrlField\n canDisplayPointField\n isImportScheduled\n importProgress {\n id\n hasForecast\n status\n total\n succeeded\n estimatedFinishTime\n actualFinishTime\n inQueue\n numberOfJobsAheadInQueue\n sendEmail\n }\n isUpdateScheduled\n updateProgress {\n id\n hasForecast\n status\n total\n succeeded\n estimatedFinishTime\n actualFinishTime\n inQueue\n numberOfJobsAheadInQueue\n sendEmail\n }\n importedDataCount\n importedDataGeocodingRate\n regionCount: importedDataCountOfAreas(\n analyticalAreaType: european_electoral_region\n )\n constituencyCount: importedDataCountOfAreas(\n analyticalAreaType: parliamentary_constituency\n )\n ladCount: importedDataCountOfAreas(analyticalAreaType: admin_district)\n wardCount: importedDataCountOfAreas(analyticalAreaType: admin_ward)\n fieldDefinitions {\n label\n value\n description\n editable\n }\n updateMapping {\n source\n sourcePath\n destinationColumn\n }\n sharingPermissions {\n id\n }\n organisation {\n id\n name\n }\n }\n }\n": types.ExternalDataSourceInspectPageDocument, "\n mutation DeleteUpdateConfig($id: String!) {\n deleteExternalDataSource(data: { id: $id }) {\n id\n }\n }\n": types.DeleteUpdateConfigDocument, + "\n mutation DeleteRecords($externalDataSourceId: String!) {\n deleteAllRecords(\n externalDataSourceId: $externalDataSourceId\n ) {\n id\n }\n }\n ": types.DeleteRecordsDocument, "\n query ManageSourceSharing($externalDataSourceId: ID!) {\n externalDataSource(pk: $externalDataSourceId) {\n sharingPermissions {\n id\n organisationId\n organisation {\n name\n }\n externalDataSourceId\n visibilityRecordCoordinates\n visibilityRecordDetails\n deleted\n }\n }\n }\n ": types.ManageSourceSharingDocument, "\n mutation UpdateSourceSharingObject(\n $data: SharingPermissionCUDInput!\n ) {\n updateSharingPermission(data: $data) {\n id\n organisationId\n externalDataSourceId\n visibilityRecordCoordinates\n visibilityRecordDetails\n deleted\n }\n }\n ": types.UpdateSourceSharingObjectDocument, "\n mutation DeleteSourceSharingObject($pk: String!) {\n deleteSharingPermission(data: { id: $pk }) {\n id\n }\n }\n ": types.DeleteSourceSharingObjectDocument, "\n mutation ImportData($id: String!) {\n importAll(externalDataSourceId: $id) {\n id\n externalDataSource {\n importedDataCount\n importProgress {\n status\n hasForecast\n id\n total\n succeeded\n failed\n estimatedFinishTime\n inQueue\n }\n }\n }\n }\n ": types.ImportDataDocument, + "\n mutation CancelImport($id: String!, $requestId: String!) {\n cancelImport(externalDataSourceId: $id, requestId: $requestId) {\n id\n }\n }\n ": types.CancelImportDocument, "\n query ExternalDataSourceName($externalDataSourceId: ID!) {\n externalDataSource(pk: $externalDataSourceId) {\n name\n crmType\n dataType\n name\n remoteUrl\n }\n }\n": types.ExternalDataSourceNameDocument, "\n mutation ShareDataSources(\n $fromOrgId: String!\n $permissions: [SharingPermissionInput!]!\n ) {\n updateSharingPermissions(\n fromOrgId: $fromOrgId\n permissions: $permissions\n ) {\n id\n sharingPermissions {\n id\n organisationId\n externalDataSourceId\n visibilityRecordCoordinates\n visibilityRecordDetails\n deleted\n }\n }\n }\n ": types.ShareDataSourcesDocument, "\n query YourSourcesForSharing {\n myOrganisations {\n id\n name\n externalDataSources {\n id\n name\n crmType\n importedDataCount\n dataType\n fieldDefinitions {\n label\n editable\n }\n organisationId\n sharingPermissions {\n id\n organisationId\n externalDataSourceId\n visibilityRecordCoordinates\n visibilityRecordDetails\n deleted\n }\n }\n }\n }\n": types.YourSourcesForSharingDocument, @@ -86,7 +88,7 @@ const documents = { "\n query GetHubHomepageJson($hostname: String!) {\n hubPageByPath(hostname: $hostname) {\n puckJsonContent\n }\n }\n": types.GetHubHomepageJsonDocument, "\n query HubListDataSources($currentOrganisationId: ID!) {\n myOrganisations(filters: { id: $currentOrganisationId }) {\n id\n externalDataSources {\n id\n name\n dataType\n }\n }\n }\n": types.HubListDataSourcesDocument, "\n mutation AddMember($externalDataSourceId: String!, $email: String!, $postcode: String!, $customFields: JSON!, $tags: [String!]!) {\n addMember(externalDataSourceId: $externalDataSourceId, email: $email, postcode: $postcode, customFields: $customFields, tags: $tags)\n }\n": types.AddMemberDocument, - "\n mutation UpdateExternalDataSource($input: ExternalDataSourceInput!) {\n updateExternalDataSource(input: $input) {\n id\n name\n geographyColumn\n geographyColumnType\n postcodeField\n firstNameField\n lastNameField\n emailField\n phoneField\n addressField\n canDisplayPointField\n autoImportEnabled\n autoUpdateEnabled\n updateMapping {\n source\n sourcePath\n destinationColumn\n }\n }\n }\n": types.UpdateExternalDataSourceDocument, + "\n mutation UpdateExternalDataSource($input: ExternalDataSourceInput!) {\n updateExternalDataSource(input: $input) {\n id\n name\n geographyColumn\n geographyColumnType\n geocodingConfig\n usesValidGeocodingConfig\n postcodeField\n firstNameField\n lastNameField\n emailField\n phoneField\n addressField\n canDisplayPointField\n autoImportEnabled\n autoUpdateEnabled\n updateMapping {\n source\n sourcePath\n destinationColumn\n }\n }\n }\n": types.UpdateExternalDataSourceDocument, "\n fragment MapReportLayersSummary on MapReport {\n layers {\n id\n name\n sharingPermission {\n visibilityRecordDetails\n visibilityRecordCoordinates\n organisation {\n name\n }\n }\n source {\n id\n name\n isImportScheduled\n importedDataCount\n crmType\n dataType\n organisation {\n name\n }\n }\n }\n }\n": types.MapReportLayersSummaryFragmentDoc, "\n fragment MapReportPage on MapReport {\n id\n name\n ...MapReportLayersSummary\n }\n \n": types.MapReportPageFragmentDoc, "\n query PublicUser {\n publicUser {\n id\n username\n email\n }\n }\n": types.PublicUserDocument, @@ -145,11 +147,11 @@ export function gql(source: "\n query ListOrganisations($currentOrganisationId: /** * The gql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. */ -export function gql(source: "\n query GetSourceMapping($ID: ID!) {\n externalDataSource(pk: $ID) {\n id\n autoImportEnabled\n autoUpdateEnabled\n allowUpdates\n hasWebhooks\n updateMapping {\n destinationColumn\n source\n sourcePath\n }\n fieldDefinitions {\n label\n value\n description\n editable\n }\n crmType\n geographyColumn\n geographyColumnType\n postcodeField\n firstNameField\n lastNameField\n emailField\n phoneField\n addressField\n canDisplayPointField\n }\n }\n"): (typeof documents)["\n query GetSourceMapping($ID: ID!) {\n externalDataSource(pk: $ID) {\n id\n autoImportEnabled\n autoUpdateEnabled\n allowUpdates\n hasWebhooks\n updateMapping {\n destinationColumn\n source\n sourcePath\n }\n fieldDefinitions {\n label\n value\n description\n editable\n }\n crmType\n geographyColumn\n geographyColumnType\n postcodeField\n firstNameField\n lastNameField\n emailField\n phoneField\n addressField\n canDisplayPointField\n }\n }\n"]; +export function gql(source: "\n query GetSourceMapping($ID: ID!) {\n externalDataSource(pk: $ID) {\n id\n autoImportEnabled\n autoUpdateEnabled\n allowUpdates\n hasWebhooks\n updateMapping {\n destinationColumn\n source\n sourcePath\n }\n fieldDefinitions {\n label\n value\n description\n editable\n }\n crmType\n geographyColumn\n geographyColumnType\n geocodingConfig\n usesValidGeocodingConfig\n postcodeField\n firstNameField\n lastNameField\n emailField\n phoneField\n addressField\n canDisplayPointField\n }\n }\n"): (typeof documents)["\n query GetSourceMapping($ID: ID!) {\n externalDataSource(pk: $ID) {\n id\n autoImportEnabled\n autoUpdateEnabled\n allowUpdates\n hasWebhooks\n updateMapping {\n destinationColumn\n source\n sourcePath\n }\n fieldDefinitions {\n label\n value\n description\n editable\n }\n crmType\n geographyColumn\n geographyColumnType\n geocodingConfig\n usesValidGeocodingConfig\n postcodeField\n firstNameField\n lastNameField\n emailField\n phoneField\n addressField\n canDisplayPointField\n }\n }\n"]; /** * The gql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. */ -export function gql(source: "\n query TestDataSource($input: CreateExternalDataSourceInput!) {\n testDataSource(input: $input) {\n __typename\n crmType\n fieldDefinitions {\n label\n value\n description\n editable\n }\n geographyColumn\n geographyColumnType\n healthcheck\n predefinedColumnNames\n defaultDataType\n remoteName\n allowUpdates\n defaults\n oauthCredentials\n }\n }\n"): (typeof documents)["\n query TestDataSource($input: CreateExternalDataSourceInput!) {\n testDataSource(input: $input) {\n __typename\n crmType\n fieldDefinitions {\n label\n value\n description\n editable\n }\n geographyColumn\n geographyColumnType\n healthcheck\n predefinedColumnNames\n defaultDataType\n remoteName\n allowUpdates\n defaults\n oauthCredentials\n }\n }\n"]; +export function gql(source: "\n query TestDataSource($input: CreateExternalDataSourceInput!) {\n testDataSource(input: $input) {\n __typename\n crmType\n fieldDefinitions {\n label\n value\n description\n editable\n }\n geographyColumn\n geographyColumnType\n geocodingConfig\n usesValidGeocodingConfig\n healthcheck\n predefinedColumnNames\n defaultDataType\n remoteName\n allowUpdates\n defaults\n oauthCredentials\n }\n }\n"): (typeof documents)["\n query TestDataSource($input: CreateExternalDataSourceInput!) {\n testDataSource(input: $input) {\n __typename\n crmType\n fieldDefinitions {\n label\n value\n description\n editable\n }\n geographyColumn\n geographyColumnType\n geocodingConfig\n usesValidGeocodingConfig\n healthcheck\n predefinedColumnNames\n defaultDataType\n remoteName\n allowUpdates\n defaults\n oauthCredentials\n }\n }\n"]; /** * The gql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. */ @@ -165,15 +167,19 @@ export function gql(source: "\n mutation CreateSource($input: CreateExternalDat /** * The gql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. */ -export function gql(source: "\n query AutoUpdateCreationReview($ID: ID!) {\n externalDataSource(pk: $ID) {\n id\n name\n geographyColumn\n geographyColumnType\n dataType\n crmType\n autoImportEnabled\n autoUpdateEnabled\n updateMapping {\n source\n sourcePath\n destinationColumn\n }\n jobs(pagination: { limit: 10 }) {\n lastEventAt\n status\n }\n automatedWebhooks\n webhookUrl\n ...DataSourceCard\n }\n }\n \n"): (typeof documents)["\n query AutoUpdateCreationReview($ID: ID!) {\n externalDataSource(pk: $ID) {\n id\n name\n geographyColumn\n geographyColumnType\n dataType\n crmType\n autoImportEnabled\n autoUpdateEnabled\n updateMapping {\n source\n sourcePath\n destinationColumn\n }\n jobs(pagination: { limit: 10 }) {\n lastEventAt\n status\n }\n automatedWebhooks\n webhookUrl\n ...DataSourceCard\n }\n }\n \n"]; +export function gql(source: "\n query AutoUpdateCreationReview($ID: ID!) {\n externalDataSource(pk: $ID) {\n id\n name\n geographyColumn\n geographyColumnType\n geocodingConfig\n usesValidGeocodingConfig\n dataType\n crmType\n autoImportEnabled\n autoUpdateEnabled\n updateMapping {\n source\n sourcePath\n destinationColumn\n }\n jobs(pagination: { limit: 10 }) {\n lastEventAt\n status\n }\n automatedWebhooks\n webhookUrl\n ...DataSourceCard\n }\n }\n \n"): (typeof documents)["\n query AutoUpdateCreationReview($ID: ID!) {\n externalDataSource(pk: $ID) {\n id\n name\n geographyColumn\n geographyColumnType\n geocodingConfig\n usesValidGeocodingConfig\n dataType\n crmType\n autoImportEnabled\n autoUpdateEnabled\n updateMapping {\n source\n sourcePath\n destinationColumn\n }\n jobs(pagination: { limit: 10 }) {\n lastEventAt\n status\n }\n automatedWebhooks\n webhookUrl\n ...DataSourceCard\n }\n }\n \n"]; /** * The gql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. */ -export function gql(source: "\n query ExternalDataSourceInspectPage($ID: ID!) {\n externalDataSource(pk: $ID) {\n id\n name\n dataType\n remoteUrl\n crmType\n connectionDetails {\n ... on AirtableSource {\n apiKey\n baseId\n tableId\n }\n ... on MailchimpSource {\n apiKey\n listId\n }\n ... on ActionNetworkSource {\n apiKey\n groupSlug\n }\n ... on TicketTailorSource {\n apiKey\n }\n }\n lastImportJob {\n id\n lastEventAt\n status\n }\n lastUpdateJob {\n id\n lastEventAt\n status\n }\n autoImportEnabled\n autoUpdateEnabled\n hasWebhooks\n allowUpdates\n automatedWebhooks\n webhookUrl\n webhookHealthcheck\n geographyColumn\n geographyColumnType\n postcodeField\n firstNameField\n lastNameField\n fullNameField\n emailField\n phoneField\n addressField\n titleField\n descriptionField\n imageField\n startTimeField\n endTimeField\n publicUrlField\n socialUrlField\n canDisplayPointField\n isImportScheduled\n importProgress {\n id\n hasForecast\n status\n total\n succeeded\n estimatedFinishTime\n actualFinishTime\n inQueue\n numberOfJobsAheadInQueue\n sendEmail\n }\n isUpdateScheduled\n updateProgress {\n id\n hasForecast\n status\n total\n succeeded\n estimatedFinishTime\n actualFinishTime\n inQueue\n numberOfJobsAheadInQueue\n sendEmail\n }\n importedDataCount\n fieldDefinitions {\n label\n value\n description\n editable\n }\n updateMapping {\n source\n sourcePath\n destinationColumn\n }\n sharingPermissions {\n id\n }\n organisation {\n id\n name\n }\n }\n }\n"): (typeof documents)["\n query ExternalDataSourceInspectPage($ID: ID!) {\n externalDataSource(pk: $ID) {\n id\n name\n dataType\n remoteUrl\n crmType\n connectionDetails {\n ... on AirtableSource {\n apiKey\n baseId\n tableId\n }\n ... on MailchimpSource {\n apiKey\n listId\n }\n ... on ActionNetworkSource {\n apiKey\n groupSlug\n }\n ... on TicketTailorSource {\n apiKey\n }\n }\n lastImportJob {\n id\n lastEventAt\n status\n }\n lastUpdateJob {\n id\n lastEventAt\n status\n }\n autoImportEnabled\n autoUpdateEnabled\n hasWebhooks\n allowUpdates\n automatedWebhooks\n webhookUrl\n webhookHealthcheck\n geographyColumn\n geographyColumnType\n postcodeField\n firstNameField\n lastNameField\n fullNameField\n emailField\n phoneField\n addressField\n titleField\n descriptionField\n imageField\n startTimeField\n endTimeField\n publicUrlField\n socialUrlField\n canDisplayPointField\n isImportScheduled\n importProgress {\n id\n hasForecast\n status\n total\n succeeded\n estimatedFinishTime\n actualFinishTime\n inQueue\n numberOfJobsAheadInQueue\n sendEmail\n }\n isUpdateScheduled\n updateProgress {\n id\n hasForecast\n status\n total\n succeeded\n estimatedFinishTime\n actualFinishTime\n inQueue\n numberOfJobsAheadInQueue\n sendEmail\n }\n importedDataCount\n fieldDefinitions {\n label\n value\n description\n editable\n }\n updateMapping {\n source\n sourcePath\n destinationColumn\n }\n sharingPermissions {\n id\n }\n organisation {\n id\n name\n }\n }\n }\n"]; +export function gql(source: "\n query ExternalDataSourceInspectPage($ID: ID!) {\n externalDataSource(pk: $ID) {\n id\n name\n dataType\n remoteUrl\n crmType\n connectionDetails {\n ... on AirtableSource {\n apiKey\n baseId\n tableId\n }\n ... on MailchimpSource {\n apiKey\n listId\n }\n ... on ActionNetworkSource {\n apiKey\n groupSlug\n }\n ... on TicketTailorSource {\n apiKey\n }\n }\n lastImportJob {\n id\n lastEventAt\n status\n }\n lastUpdateJob {\n id\n lastEventAt\n status\n }\n autoImportEnabled\n autoUpdateEnabled\n hasWebhooks\n allowUpdates\n automatedWebhooks\n webhookUrl\n webhookHealthcheck\n geographyColumn\n geographyColumnType\n geocodingConfig\n usesValidGeocodingConfig\n postcodeField\n firstNameField\n lastNameField\n fullNameField\n emailField\n phoneField\n addressField\n titleField\n descriptionField\n imageField\n startTimeField\n endTimeField\n publicUrlField\n socialUrlField\n canDisplayPointField\n isImportScheduled\n importProgress {\n id\n hasForecast\n status\n total\n succeeded\n estimatedFinishTime\n actualFinishTime\n inQueue\n numberOfJobsAheadInQueue\n sendEmail\n }\n isUpdateScheduled\n updateProgress {\n id\n hasForecast\n status\n total\n succeeded\n estimatedFinishTime\n actualFinishTime\n inQueue\n numberOfJobsAheadInQueue\n sendEmail\n }\n importedDataCount\n importedDataGeocodingRate\n regionCount: importedDataCountOfAreas(\n analyticalAreaType: european_electoral_region\n )\n constituencyCount: importedDataCountOfAreas(\n analyticalAreaType: parliamentary_constituency\n )\n ladCount: importedDataCountOfAreas(analyticalAreaType: admin_district)\n wardCount: importedDataCountOfAreas(analyticalAreaType: admin_ward)\n fieldDefinitions {\n label\n value\n description\n editable\n }\n updateMapping {\n source\n sourcePath\n destinationColumn\n }\n sharingPermissions {\n id\n }\n organisation {\n id\n name\n }\n }\n }\n"): (typeof documents)["\n query ExternalDataSourceInspectPage($ID: ID!) {\n externalDataSource(pk: $ID) {\n id\n name\n dataType\n remoteUrl\n crmType\n connectionDetails {\n ... on AirtableSource {\n apiKey\n baseId\n tableId\n }\n ... on MailchimpSource {\n apiKey\n listId\n }\n ... on ActionNetworkSource {\n apiKey\n groupSlug\n }\n ... on TicketTailorSource {\n apiKey\n }\n }\n lastImportJob {\n id\n lastEventAt\n status\n }\n lastUpdateJob {\n id\n lastEventAt\n status\n }\n autoImportEnabled\n autoUpdateEnabled\n hasWebhooks\n allowUpdates\n automatedWebhooks\n webhookUrl\n webhookHealthcheck\n geographyColumn\n geographyColumnType\n geocodingConfig\n usesValidGeocodingConfig\n postcodeField\n firstNameField\n lastNameField\n fullNameField\n emailField\n phoneField\n addressField\n titleField\n descriptionField\n imageField\n startTimeField\n endTimeField\n publicUrlField\n socialUrlField\n canDisplayPointField\n isImportScheduled\n importProgress {\n id\n hasForecast\n status\n total\n succeeded\n estimatedFinishTime\n actualFinishTime\n inQueue\n numberOfJobsAheadInQueue\n sendEmail\n }\n isUpdateScheduled\n updateProgress {\n id\n hasForecast\n status\n total\n succeeded\n estimatedFinishTime\n actualFinishTime\n inQueue\n numberOfJobsAheadInQueue\n sendEmail\n }\n importedDataCount\n importedDataGeocodingRate\n regionCount: importedDataCountOfAreas(\n analyticalAreaType: european_electoral_region\n )\n constituencyCount: importedDataCountOfAreas(\n analyticalAreaType: parliamentary_constituency\n )\n ladCount: importedDataCountOfAreas(analyticalAreaType: admin_district)\n wardCount: importedDataCountOfAreas(analyticalAreaType: admin_ward)\n fieldDefinitions {\n label\n value\n description\n editable\n }\n updateMapping {\n source\n sourcePath\n destinationColumn\n }\n sharingPermissions {\n id\n }\n organisation {\n id\n name\n }\n }\n }\n"]; /** * The gql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. */ export function gql(source: "\n mutation DeleteUpdateConfig($id: String!) {\n deleteExternalDataSource(data: { id: $id }) {\n id\n }\n }\n"): (typeof documents)["\n mutation DeleteUpdateConfig($id: String!) {\n deleteExternalDataSource(data: { id: $id }) {\n id\n }\n }\n"]; +/** + * The gql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. + */ +export function gql(source: "\n mutation DeleteRecords($externalDataSourceId: String!) {\n deleteAllRecords(\n externalDataSourceId: $externalDataSourceId\n ) {\n id\n }\n }\n "): (typeof documents)["\n mutation DeleteRecords($externalDataSourceId: String!) {\n deleteAllRecords(\n externalDataSourceId: $externalDataSourceId\n ) {\n id\n }\n }\n "]; /** * The gql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. */ @@ -190,6 +196,10 @@ export function gql(source: "\n mutation DeleteSourceSharingObject($pk: * The gql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. */ export function gql(source: "\n mutation ImportData($id: String!) {\n importAll(externalDataSourceId: $id) {\n id\n externalDataSource {\n importedDataCount\n importProgress {\n status\n hasForecast\n id\n total\n succeeded\n failed\n estimatedFinishTime\n inQueue\n }\n }\n }\n }\n "): (typeof documents)["\n mutation ImportData($id: String!) {\n importAll(externalDataSourceId: $id) {\n id\n externalDataSource {\n importedDataCount\n importProgress {\n status\n hasForecast\n id\n total\n succeeded\n failed\n estimatedFinishTime\n inQueue\n }\n }\n }\n }\n "]; +/** + * The gql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. + */ +export function gql(source: "\n mutation CancelImport($id: String!, $requestId: String!) {\n cancelImport(externalDataSourceId: $id, requestId: $requestId) {\n id\n }\n }\n "): (typeof documents)["\n mutation CancelImport($id: String!, $requestId: String!) {\n cancelImport(externalDataSourceId: $id, requestId: $requestId) {\n id\n }\n }\n "]; /** * The gql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. */ @@ -397,7 +407,7 @@ export function gql(source: "\n mutation AddMember($externalDataSourceId: Strin /** * The gql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. */ -export function gql(source: "\n mutation UpdateExternalDataSource($input: ExternalDataSourceInput!) {\n updateExternalDataSource(input: $input) {\n id\n name\n geographyColumn\n geographyColumnType\n postcodeField\n firstNameField\n lastNameField\n emailField\n phoneField\n addressField\n canDisplayPointField\n autoImportEnabled\n autoUpdateEnabled\n updateMapping {\n source\n sourcePath\n destinationColumn\n }\n }\n }\n"): (typeof documents)["\n mutation UpdateExternalDataSource($input: ExternalDataSourceInput!) {\n updateExternalDataSource(input: $input) {\n id\n name\n geographyColumn\n geographyColumnType\n postcodeField\n firstNameField\n lastNameField\n emailField\n phoneField\n addressField\n canDisplayPointField\n autoImportEnabled\n autoUpdateEnabled\n updateMapping {\n source\n sourcePath\n destinationColumn\n }\n }\n }\n"]; +export function gql(source: "\n mutation UpdateExternalDataSource($input: ExternalDataSourceInput!) {\n updateExternalDataSource(input: $input) {\n id\n name\n geographyColumn\n geographyColumnType\n geocodingConfig\n usesValidGeocodingConfig\n postcodeField\n firstNameField\n lastNameField\n emailField\n phoneField\n addressField\n canDisplayPointField\n autoImportEnabled\n autoUpdateEnabled\n updateMapping {\n source\n sourcePath\n destinationColumn\n }\n }\n }\n"): (typeof documents)["\n mutation UpdateExternalDataSource($input: ExternalDataSourceInput!) {\n updateExternalDataSource(input: $input) {\n id\n name\n geographyColumn\n geographyColumnType\n geocodingConfig\n usesValidGeocodingConfig\n postcodeField\n firstNameField\n lastNameField\n emailField\n phoneField\n addressField\n canDisplayPointField\n autoImportEnabled\n autoUpdateEnabled\n updateMapping {\n source\n sourcePath\n destinationColumn\n }\n }\n }\n"]; /** * The gql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. */ diff --git a/nextjs/src/__generated__/graphql.ts b/nextjs/src/__generated__/graphql.ts index 3c21bc677..7c46eed9d 100644 --- a/nextjs/src/__generated__/graphql.ts +++ b/nextjs/src/__generated__/graphql.ts @@ -86,6 +86,7 @@ export type ActionNetworkSource = Analytics & { fieldDefinitions?: Maybe>; firstNameField?: Maybe; fullNameField?: Maybe; + geocodingConfig: Scalars['JSON']['output']; geographyColumn?: Maybe; geographyColumnType: GeographyTypes; groupSlug: Scalars['String']['output']; @@ -105,6 +106,10 @@ export type ActionNetworkSource = Analytics & { importedDataCountForArea?: Maybe; importedDataCountForConstituency?: Maybe; importedDataCountForConstituency2024?: Maybe; + importedDataCountLocated: Scalars['Int']['output']; + importedDataCountOfAreas: Scalars['Int']['output']; + importedDataCountUnlocated: Scalars['Int']['output']; + importedDataGeocodingRate: Scalars['Float']['output']; introspectFields: Scalars['Boolean']['output']; isImportScheduled: Scalars['Boolean']['output']; isUpdateScheduled: Scalars['Boolean']['output']; @@ -131,6 +136,7 @@ export type ActionNetworkSource = Analytics & { titleField?: Maybe; updateMapping?: Maybe>; updateProgress?: Maybe; + usesValidGeocodingConfig: Scalars['Boolean']['output']; webhookHealthcheck: Scalars['Boolean']['output']; webhookUrl: Scalars['String']['output']; }; @@ -175,6 +181,13 @@ export type ActionNetworkSourceImportedDataCountForConstituency2024Args = { }; +/** An Action Network member list. */ +export type ActionNetworkSourceImportedDataCountOfAreasArgs = { + analyticalAreaType: AnalyticalAreaType; + layerIds?: InputMaybe>; +}; + + /** An Action Network member list. */ export type ActionNetworkSourceJobsArgs = { filters?: InputMaybe; @@ -201,6 +214,7 @@ export type ActionNetworkSourceInput = { endTimeField?: InputMaybe; firstNameField?: InputMaybe; fullNameField?: InputMaybe; + geocodingConfig?: InputMaybe; geographyColumn?: InputMaybe; geographyColumnType?: InputMaybe; groupSlug: Scalars['String']['input']; @@ -243,6 +257,7 @@ export type AirtableSource = Analytics & { fieldDefinitions?: Maybe>; firstNameField?: Maybe; fullNameField?: Maybe; + geocodingConfig: Scalars['JSON']['output']; geographyColumn?: Maybe; geographyColumnType: GeographyTypes; hasWebhooks: Scalars['Boolean']['output']; @@ -261,6 +276,10 @@ export type AirtableSource = Analytics & { importedDataCountForArea?: Maybe; importedDataCountForConstituency?: Maybe; importedDataCountForConstituency2024?: Maybe; + importedDataCountLocated: Scalars['Int']['output']; + importedDataCountOfAreas: Scalars['Int']['output']; + importedDataCountUnlocated: Scalars['Int']['output']; + importedDataGeocodingRate: Scalars['Float']['output']; introspectFields: Scalars['Boolean']['output']; isImportScheduled: Scalars['Boolean']['output']; isUpdateScheduled: Scalars['Boolean']['output']; @@ -288,6 +307,7 @@ export type AirtableSource = Analytics & { titleField?: Maybe; updateMapping?: Maybe>; updateProgress?: Maybe; + usesValidGeocodingConfig: Scalars['Boolean']['output']; webhookHealthcheck: Scalars['Boolean']['output']; webhookUrl: Scalars['String']['output']; }; @@ -332,6 +352,13 @@ export type AirtableSourceImportedDataCountForConstituency2024Args = { }; +/** An Airtable table. */ +export type AirtableSourceImportedDataCountOfAreasArgs = { + analyticalAreaType: AnalyticalAreaType; + layerIds?: InputMaybe>; +}; + + /** An Airtable table. */ export type AirtableSourceJobsArgs = { filters?: InputMaybe; @@ -360,6 +387,7 @@ export type AirtableSourceInput = { endTimeField?: InputMaybe; firstNameField?: InputMaybe; fullNameField?: InputMaybe; + geocodingConfig?: InputMaybe; geographyColumn?: InputMaybe; geographyColumnType?: InputMaybe; id?: InputMaybe; @@ -385,8 +413,11 @@ export enum AnalyticalAreaType { AdminWard = 'admin_ward', Country = 'country', EuropeanElectoralRegion = 'european_electoral_region', + Lsoa = 'lsoa', + Msoa = 'msoa', ParliamentaryConstituency = 'parliamentary_constituency', - ParliamentaryConstituency_2024 = 'parliamentary_constituency_2024' + ParliamentaryConstituency_2024 = 'parliamentary_constituency_2024', + Postcode = 'postcode' } export type Analytics = { @@ -401,6 +432,10 @@ export type Analytics = { importedDataCountForArea?: Maybe; importedDataCountForConstituency?: Maybe; importedDataCountForConstituency2024?: Maybe; + importedDataCountLocated: Scalars['Int']['output']; + importedDataCountOfAreas: Scalars['Int']['output']; + importedDataCountUnlocated: Scalars['Int']['output']; + importedDataGeocodingRate: Scalars['Float']['output']; }; @@ -436,7 +471,13 @@ export type AnalyticsImportedDataCountForConstituency2024Args = { gss: Scalars['String']['input']; }; -/** Area(id, mapit_id, gss, name, area_type, geometry, polygon, point) */ + +export type AnalyticsImportedDataCountOfAreasArgs = { + analyticalAreaType: AnalyticalAreaType; + layerIds?: InputMaybe>; +}; + +/** Area(id, mapit_id, mapit_type, mapit_generation_low, mapit_generation_high, gss, name, mapit_all_names, area_type, geometry, polygon, point) */ export type Area = { __typename?: 'Area'; areaType: AreaType; @@ -459,43 +500,43 @@ export type Area = { }; -/** Area(id, mapit_id, gss, name, area_type, geometry, polygon, point) */ +/** Area(id, mapit_id, mapit_type, mapit_generation_low, mapit_generation_high, gss, name, mapit_all_names, area_type, geometry, polygon, point) */ export type AreaDataArgs = { filters?: InputMaybe; }; -/** Area(id, mapit_id, gss, name, area_type, geometry, polygon, point) */ +/** Area(id, mapit_id, mapit_type, mapit_generation_low, mapit_generation_high, gss, name, mapit_all_names, area_type, geometry, polygon, point) */ export type AreaDatumArgs = { filters?: InputMaybe; }; -/** Area(id, mapit_id, gss, name, area_type, geometry, polygon, point) */ +/** Area(id, mapit_id, mapit_type, mapit_generation_low, mapit_generation_high, gss, name, mapit_all_names, area_type, geometry, polygon, point) */ export type AreaGenericDataForHubArgs = { hostname: Scalars['String']['input']; }; -/** Area(id, mapit_id, gss, name, area_type, geometry, polygon, point) */ +/** Area(id, mapit_id, mapit_type, mapit_generation_low, mapit_generation_high, gss, name, mapit_all_names, area_type, geometry, polygon, point) */ export type AreaPeopleArgs = { filters?: InputMaybe; }; -/** Area(id, mapit_id, gss, name, area_type, geometry, polygon, point) */ +/** Area(id, mapit_id, mapit_type, mapit_generation_low, mapit_generation_high, gss, name, mapit_all_names, area_type, geometry, polygon, point) */ export type AreaPersonArgs = { filters?: InputMaybe; }; -/** Area(id, mapit_id, gss, name, area_type, geometry, polygon, point) */ +/** Area(id, mapit_id, mapit_type, mapit_generation_low, mapit_generation_high, gss, name, mapit_all_names, area_type, geometry, polygon, point) */ export type AreaPointArgs = { withParentData?: Scalars['Boolean']['input']; }; -/** Area(id, mapit_id, gss, name, area_type, geometry, polygon, point) */ +/** Area(id, mapit_id, mapit_type, mapit_generation_low, mapit_generation_high, gss, name, mapit_all_names, area_type, geometry, polygon, point) */ export type AreaPolygonArgs = { withParentData?: Scalars['Boolean']['input']; }; @@ -524,6 +565,12 @@ export type AreaType = { name: Scalars['String']['output']; }; +export type AreaTypeFilter = { + __typename?: 'AreaTypeFilter'; + lihAreaType?: Maybe; + mapitAreaTypes?: Maybe>; +}; + export type AuthenticatedPostcodeQueryResponse = { __typename?: 'AuthenticatedPostcodeQueryResponse'; constituency?: Maybe; @@ -779,6 +826,7 @@ export type EditableGoogleSheetsSource = Analytics & { fieldDefinitions?: Maybe>; firstNameField?: Maybe; fullNameField?: Maybe; + geocodingConfig: Scalars['JSON']['output']; geographyColumn?: Maybe; geographyColumnType: GeographyTypes; hasWebhooks: Scalars['Boolean']['output']; @@ -797,6 +845,10 @@ export type EditableGoogleSheetsSource = Analytics & { importedDataCountForArea?: Maybe; importedDataCountForConstituency?: Maybe; importedDataCountForConstituency2024?: Maybe; + importedDataCountLocated: Scalars['Int']['output']; + importedDataCountOfAreas: Scalars['Int']['output']; + importedDataCountUnlocated: Scalars['Int']['output']; + importedDataGeocodingRate: Scalars['Float']['output']; introspectFields: Scalars['Boolean']['output']; isImportScheduled: Scalars['Boolean']['output']; isUpdateScheduled: Scalars['Boolean']['output']; @@ -825,6 +877,7 @@ export type EditableGoogleSheetsSource = Analytics & { titleField?: Maybe; updateMapping?: Maybe>; updateProgress?: Maybe; + usesValidGeocodingConfig: Scalars['Boolean']['output']; webhookHealthcheck: Scalars['Boolean']['output']; webhookUrl: Scalars['String']['output']; }; @@ -869,6 +922,13 @@ export type EditableGoogleSheetsSourceImportedDataCountForConstituency2024Args = }; +/** An editable Google Sheet */ +export type EditableGoogleSheetsSourceImportedDataCountOfAreasArgs = { + analyticalAreaType: AnalyticalAreaType; + layerIds?: InputMaybe>; +}; + + /** An editable Google Sheet */ export type EditableGoogleSheetsSourceJobsArgs = { filters?: InputMaybe; @@ -894,6 +954,7 @@ export type EditableGoogleSheetsSourceInput = { endTimeField?: InputMaybe; firstNameField?: InputMaybe; fullNameField?: InputMaybe; + geocodingConfig?: InputMaybe; geographyColumn?: InputMaybe; geographyColumnType?: InputMaybe; id?: InputMaybe; @@ -1031,6 +1092,7 @@ export type ExternalDataSource = Analytics & { fieldDefinitions?: Maybe>; firstNameField?: Maybe; fullNameField?: Maybe; + geocodingConfig: Scalars['JSON']['output']; geographyColumn?: Maybe; geographyColumnType: GeographyTypes; hasWebhooks: Scalars['Boolean']['output']; @@ -1049,6 +1111,10 @@ export type ExternalDataSource = Analytics & { importedDataCountForArea?: Maybe; importedDataCountForConstituency?: Maybe; importedDataCountForConstituency2024?: Maybe; + importedDataCountLocated: Scalars['Int']['output']; + importedDataCountOfAreas: Scalars['Int']['output']; + importedDataCountUnlocated: Scalars['Int']['output']; + importedDataGeocodingRate: Scalars['Float']['output']; introspectFields: Scalars['Boolean']['output']; isImportScheduled: Scalars['Boolean']['output']; isUpdateScheduled: Scalars['Boolean']['output']; @@ -1075,6 +1141,7 @@ export type ExternalDataSource = Analytics & { titleField?: Maybe; updateMapping?: Maybe>; updateProgress?: Maybe; + usesValidGeocodingConfig: Scalars['Boolean']['output']; webhookHealthcheck: Scalars['Boolean']['output']; webhookUrl: Scalars['String']['output']; }; @@ -1143,6 +1210,17 @@ export type ExternalDataSourceImportedDataCountForConstituency2024Args = { }; +/** + * A third-party data source that can be read and optionally written back to. + * E.g. Google Sheet or an Action Network table. + * This class is to be subclassed by specific data source types. + */ +export type ExternalDataSourceImportedDataCountOfAreasArgs = { + analyticalAreaType: AnalyticalAreaType; + layerIds?: InputMaybe>; +}; + + /** * A third-party data source that can be read and optionally written back to. * E.g. Google Sheet or an Action Network table. @@ -1199,6 +1277,7 @@ export type ExternalDataSourceInput = { endTimeField?: InputMaybe; firstNameField?: InputMaybe; fullNameField?: InputMaybe; + geocodingConfig?: InputMaybe; geographyColumn?: InputMaybe; geographyColumnType?: InputMaybe; id?: InputMaybe; @@ -1283,6 +1362,8 @@ export enum GeoJsonTypes { export enum GeographyTypes { Address = 'ADDRESS', AdminDistrict = 'ADMIN_DISTRICT', + Area = 'AREA', + Coordinates = 'COORDINATES', ParliamentaryConstituency = 'PARLIAMENTARY_CONSTITUENCY', ParliamentaryConstituency_2024 = 'PARLIAMENTARY_CONSTITUENCY_2024', Postcode = 'POSTCODE', @@ -1294,6 +1375,7 @@ export type GroupedData = { __typename?: 'GroupedData'; areaData?: Maybe; areaType?: Maybe; + areaTypeFilter?: Maybe; gss?: Maybe; gssArea?: Maybe; importedData?: Maybe; @@ -1303,7 +1385,7 @@ export type GroupedData = { export type GroupedDataCount = { __typename?: 'GroupedDataCount'; areaData?: Maybe; - areaType?: Maybe; + areaTypeFilter?: Maybe; count: Scalars['Int']['output']; gss?: Maybe; gssArea?: Maybe; @@ -1313,7 +1395,7 @@ export type GroupedDataCount = { export type GroupedDataCountWithBreakdown = { __typename?: 'GroupedDataCountWithBreakdown'; areaData?: Maybe; - areaType?: Maybe; + areaTypeFilter?: Maybe; count: Scalars['Int']['output']; gss?: Maybe; gssArea?: Maybe; @@ -1491,6 +1573,7 @@ export type MailChimpSourceInput = { endTimeField?: InputMaybe; firstNameField?: InputMaybe; fullNameField?: InputMaybe; + geocodingConfig?: InputMaybe; geographyColumn?: InputMaybe; geographyColumnType?: InputMaybe; id?: InputMaybe; @@ -1533,6 +1616,7 @@ export type MailchimpSource = Analytics & { fieldDefinitions?: Maybe>; firstNameField?: Maybe; fullNameField?: Maybe; + geocodingConfig: Scalars['JSON']['output']; geographyColumn?: Maybe; geographyColumnType: GeographyTypes; hasWebhooks: Scalars['Boolean']['output']; @@ -1551,6 +1635,10 @@ export type MailchimpSource = Analytics & { importedDataCountForArea?: Maybe; importedDataCountForConstituency?: Maybe; importedDataCountForConstituency2024?: Maybe; + importedDataCountLocated: Scalars['Int']['output']; + importedDataCountOfAreas: Scalars['Int']['output']; + importedDataCountUnlocated: Scalars['Int']['output']; + importedDataGeocodingRate: Scalars['Float']['output']; introspectFields: Scalars['Boolean']['output']; isImportScheduled: Scalars['Boolean']['output']; isUpdateScheduled: Scalars['Boolean']['output']; @@ -1579,6 +1667,7 @@ export type MailchimpSource = Analytics & { titleField?: Maybe; updateMapping?: Maybe>; updateProgress?: Maybe; + usesValidGeocodingConfig: Scalars['Boolean']['output']; webhookHealthcheck: Scalars['Boolean']['output']; webhookUrl: Scalars['String']['output']; }; @@ -1623,6 +1712,13 @@ export type MailchimpSourceImportedDataCountForConstituency2024Args = { }; +/** A Mailchimp list. */ +export type MailchimpSourceImportedDataCountOfAreasArgs = { + analyticalAreaType: AnalyticalAreaType; + layerIds?: InputMaybe>; +}; + + /** A Mailchimp list. */ export type MailchimpSourceJobsArgs = { filters?: InputMaybe; @@ -1676,6 +1772,10 @@ export type MapReport = Analytics & { importedDataCountForArea?: Maybe; importedDataCountForConstituency?: Maybe; importedDataCountForConstituency2024?: Maybe; + importedDataCountLocated: Scalars['Int']['output']; + importedDataCountOfAreas: Scalars['Int']['output']; + importedDataCountUnlocated: Scalars['Int']['output']; + importedDataGeocodingRate: Scalars['Float']['output']; lastUpdate: Scalars['DateTime']['output']; layers: Array; name: Scalars['String']['output']; @@ -1723,6 +1823,13 @@ export type MapReportImportedDataCountForConstituency2024Args = { gss: Scalars['String']['input']; }; + +/** MapReport(polymorphic_ctype, id, organisation, name, slug, description, created_at, last_update, public, report_ptr, layers, display_options) */ +export type MapReportImportedDataCountOfAreasArgs = { + analyticalAreaType: AnalyticalAreaType; + layerIds?: InputMaybe>; +}; + /** MapReport(polymorphic_ctype, id, organisation, name, slug, description, created_at, last_update, public, report_ptr, layers, display_options) */ export type MapReportInput = { createdAt?: InputMaybe; @@ -1797,12 +1904,14 @@ export type MultiPolygonGeometry = { export type Mutation = { __typename?: 'Mutation'; addMember: Scalars['Boolean']['output']; + cancelImport: ExternalDataSourceAction; createApiToken: ApiToken; createChildPage: HubPage; createExternalDataSource: CreateExternalDataSourceOutput; createMapReport: CreateMapReportPayload; createOrganisation: Membership; createSharingPermission: SharingPermission; + deleteAllRecords: ExternalDataSource; deleteExternalDataSource: ExternalDataSource; deleteMapReport: MapReport; deletePage: Scalars['Boolean']['output']; @@ -1811,15 +1920,25 @@ export type Mutation = { enableWebhook: ExternalDataSource; importAll: ExternalDataSourceAction; /** + * Change user password without old password. + * + * Receive the token that was sent by email. * - * Override library mutation to add copious logging. + * If token and new passwords are valid, update user password and in + * case of using refresh tokens, revoke all of them. + * + * Also, if user has not been verified yet, verify it. * */ performPasswordReset: MutationNormalOutput; refreshWebhooks: ExternalDataSource; /** + * Send password reset email. * - * Override library mutation to add copious logging. + * For non verified users, send an activation email instead. + * + * If there is no user with the requested email, a successful response + * is returned. * */ requestPasswordReset: MutationNormalOutput; @@ -1877,6 +1996,12 @@ export type MutationAddMemberArgs = { }; +export type MutationCancelImportArgs = { + externalDataSourceId: Scalars['String']['input']; + requestId: Scalars['String']['input']; +}; + + export type MutationCreateApiTokenArgs = { expiryDays?: Scalars['Int']['input']; }; @@ -1908,6 +2033,11 @@ export type MutationCreateSharingPermissionArgs = { }; +export type MutationDeleteAllRecordsArgs = { + externalDataSourceId: Scalars['String']['input']; +}; + + export type MutationDeleteExternalDataSourceArgs = { data: IdObject; }; @@ -2247,6 +2377,7 @@ export type PostcodesIoResult = { }; export enum ProcrastinateJobStatus { + Cancelled = 'cancelled', Doing = 'doing', Failed = 'failed', Succeeded = 'succeeded', @@ -2548,6 +2679,7 @@ export type SharedDataSource = Analytics & { endTimeField?: Maybe; firstNameField?: Maybe; fullNameField?: Maybe; + geocodingConfig: Scalars['JSON']['output']; geographyColumn?: Maybe; geographyColumnType: GeographyTypes; hasWebhooks: Scalars['Boolean']['output']; @@ -2565,6 +2697,10 @@ export type SharedDataSource = Analytics & { importedDataCountForArea?: Maybe; importedDataCountForConstituency?: Maybe; importedDataCountForConstituency2024?: Maybe; + importedDataCountLocated: Scalars['Int']['output']; + importedDataCountOfAreas: Scalars['Int']['output']; + importedDataCountUnlocated: Scalars['Int']['output']; + importedDataGeocodingRate: Scalars['Float']['output']; introspectFields: Scalars['Boolean']['output']; isImportScheduled: Scalars['Boolean']['output']; isUpdateScheduled: Scalars['Boolean']['output']; @@ -2582,6 +2718,7 @@ export type SharedDataSource = Analytics & { startTimeField?: Maybe; titleField?: Maybe; updateProgress?: Maybe; + usesValidGeocodingConfig: Scalars['Boolean']['output']; }; @@ -2647,6 +2784,17 @@ export type SharedDataSourceImportedDataCountForConstituency2024Args = { gss: Scalars['String']['input']; }; + +/** + * A third-party data source that can be read and optionally written back to. + * E.g. Google Sheet or an Action Network table. + * This class is to be subclassed by specific data source types. + */ +export type SharedDataSourceImportedDataCountOfAreasArgs = { + analyticalAreaType: AnalyticalAreaType; + layerIds?: InputMaybe>; +}; + /** SharingPermission(id, created_at, last_update, external_data_source, organisation, visibility_record_coordinates, visibility_record_details) */ export type SharingPermission = { __typename?: 'SharingPermission'; @@ -2723,6 +2871,7 @@ export type TicketTailorSource = Analytics & { fieldDefinitions?: Maybe>; firstNameField?: Maybe; fullNameField?: Maybe; + geocodingConfig: Scalars['JSON']['output']; geographyColumn?: Maybe; geographyColumnType: GeographyTypes; hasWebhooks: Scalars['Boolean']['output']; @@ -2741,6 +2890,10 @@ export type TicketTailorSource = Analytics & { importedDataCountForArea?: Maybe; importedDataCountForConstituency?: Maybe; importedDataCountForConstituency2024?: Maybe; + importedDataCountLocated: Scalars['Int']['output']; + importedDataCountOfAreas: Scalars['Int']['output']; + importedDataCountUnlocated: Scalars['Int']['output']; + importedDataGeocodingRate: Scalars['Float']['output']; introspectFields: Scalars['Boolean']['output']; isImportScheduled: Scalars['Boolean']['output']; isUpdateScheduled: Scalars['Boolean']['output']; @@ -2767,6 +2920,7 @@ export type TicketTailorSource = Analytics & { titleField?: Maybe; updateMapping?: Maybe>; updateProgress?: Maybe; + usesValidGeocodingConfig: Scalars['Boolean']['output']; webhookHealthcheck: Scalars['Boolean']['output']; webhookUrl: Scalars['String']['output']; }; @@ -2811,6 +2965,13 @@ export type TicketTailorSourceImportedDataCountForConstituency2024Args = { }; +/** Ticket Tailor box office */ +export type TicketTailorSourceImportedDataCountOfAreasArgs = { + analyticalAreaType: AnalyticalAreaType; + layerIds?: InputMaybe>; +}; + + /** Ticket Tailor box office */ export type TicketTailorSourceJobsArgs = { filters?: InputMaybe; @@ -2837,6 +2998,7 @@ export type TicketTailorSourceInput = { endTimeField?: InputMaybe; firstNameField?: InputMaybe; fullNameField?: InputMaybe; + geocodingConfig?: InputMaybe; geographyColumn?: InputMaybe; geographyColumnType?: InputMaybe; id?: InputMaybe; @@ -3024,14 +3186,14 @@ export type GetSourceMappingQueryVariables = Exact<{ }>; -export type GetSourceMappingQuery = { __typename?: 'Query', externalDataSource: { __typename?: 'ExternalDataSource', id: any, autoImportEnabled: boolean, autoUpdateEnabled: boolean, allowUpdates: boolean, hasWebhooks: boolean, crmType: CrmType, geographyColumn?: string | null, geographyColumnType: GeographyTypes, postcodeField?: string | null, firstNameField?: string | null, lastNameField?: string | null, emailField?: string | null, phoneField?: string | null, addressField?: string | null, canDisplayPointField?: string | null, updateMapping?: Array<{ __typename?: 'AutoUpdateConfig', destinationColumn: string, source: string, sourcePath: string }> | null, fieldDefinitions?: Array<{ __typename?: 'FieldDefinition', label?: string | null, value: string, description?: string | null, editable: boolean }> | null } }; +export type GetSourceMappingQuery = { __typename?: 'Query', externalDataSource: { __typename?: 'ExternalDataSource', id: any, autoImportEnabled: boolean, autoUpdateEnabled: boolean, allowUpdates: boolean, hasWebhooks: boolean, crmType: CrmType, geographyColumn?: string | null, geographyColumnType: GeographyTypes, geocodingConfig: any, usesValidGeocodingConfig: boolean, postcodeField?: string | null, firstNameField?: string | null, lastNameField?: string | null, emailField?: string | null, phoneField?: string | null, addressField?: string | null, canDisplayPointField?: string | null, updateMapping?: Array<{ __typename?: 'AutoUpdateConfig', destinationColumn: string, source: string, sourcePath: string }> | null, fieldDefinitions?: Array<{ __typename?: 'FieldDefinition', label?: string | null, value: string, description?: string | null, editable: boolean }> | null } }; export type TestDataSourceQueryVariables = Exact<{ input: CreateExternalDataSourceInput; }>; -export type TestDataSourceQuery = { __typename?: 'Query', testDataSource: { __typename: 'ExternalDataSource', crmType: CrmType, geographyColumn?: string | null, geographyColumnType: GeographyTypes, healthcheck: boolean, predefinedColumnNames: boolean, defaultDataType?: string | null, remoteName?: string | null, allowUpdates: boolean, defaults: any, oauthCredentials?: string | null, fieldDefinitions?: Array<{ __typename?: 'FieldDefinition', label?: string | null, value: string, description?: string | null, editable: boolean }> | null } }; +export type TestDataSourceQuery = { __typename?: 'Query', testDataSource: { __typename: 'ExternalDataSource', crmType: CrmType, geographyColumn?: string | null, geographyColumnType: GeographyTypes, geocodingConfig: any, usesValidGeocodingConfig: boolean, healthcheck: boolean, predefinedColumnNames: boolean, defaultDataType?: string | null, remoteName?: string | null, allowUpdates: boolean, defaults: any, oauthCredentials?: string | null, fieldDefinitions?: Array<{ __typename?: 'FieldDefinition', label?: string | null, value: string, description?: string | null, editable: boolean }> | null } }; export type GoogleSheetsOauthUrlQueryVariables = Exact<{ redirectUrl: Scalars['String']['input']; @@ -3059,14 +3221,14 @@ export type AutoUpdateCreationReviewQueryVariables = Exact<{ }>; -export type AutoUpdateCreationReviewQuery = { __typename?: 'Query', externalDataSource: { __typename?: 'ExternalDataSource', id: any, name: string, geographyColumn?: string | null, geographyColumnType: GeographyTypes, dataType: DataSourceType, crmType: CrmType, autoImportEnabled: boolean, autoUpdateEnabled: boolean, automatedWebhooks: boolean, webhookUrl: string, updateMapping?: Array<{ __typename?: 'AutoUpdateConfig', source: string, sourcePath: string, destinationColumn: string }> | null, jobs: Array<{ __typename?: 'QueueJob', lastEventAt: any, status: ProcrastinateJobStatus }>, sharingPermissions: Array<{ __typename?: 'SharingPermission', id: any, organisation: { __typename?: 'PublicOrganisation', id: string, name: string } }> } }; +export type AutoUpdateCreationReviewQuery = { __typename?: 'Query', externalDataSource: { __typename?: 'ExternalDataSource', id: any, name: string, geographyColumn?: string | null, geographyColumnType: GeographyTypes, geocodingConfig: any, usesValidGeocodingConfig: boolean, dataType: DataSourceType, crmType: CrmType, autoImportEnabled: boolean, autoUpdateEnabled: boolean, automatedWebhooks: boolean, webhookUrl: string, updateMapping?: Array<{ __typename?: 'AutoUpdateConfig', source: string, sourcePath: string, destinationColumn: string }> | null, jobs: Array<{ __typename?: 'QueueJob', lastEventAt: any, status: ProcrastinateJobStatus }>, sharingPermissions: Array<{ __typename?: 'SharingPermission', id: any, organisation: { __typename?: 'PublicOrganisation', id: string, name: string } }> } }; export type ExternalDataSourceInspectPageQueryVariables = Exact<{ ID: Scalars['ID']['input']; }>; -export type ExternalDataSourceInspectPageQuery = { __typename?: 'Query', externalDataSource: { __typename?: 'ExternalDataSource', id: any, name: string, dataType: DataSourceType, remoteUrl?: string | null, crmType: CrmType, autoImportEnabled: boolean, autoUpdateEnabled: boolean, hasWebhooks: boolean, allowUpdates: boolean, automatedWebhooks: boolean, webhookUrl: string, webhookHealthcheck: boolean, geographyColumn?: string | null, geographyColumnType: GeographyTypes, postcodeField?: string | null, firstNameField?: string | null, lastNameField?: string | null, fullNameField?: string | null, emailField?: string | null, phoneField?: string | null, addressField?: string | null, titleField?: string | null, descriptionField?: string | null, imageField?: string | null, startTimeField?: string | null, endTimeField?: string | null, publicUrlField?: string | null, socialUrlField?: string | null, canDisplayPointField?: string | null, isImportScheduled: boolean, isUpdateScheduled: boolean, importedDataCount: number, connectionDetails: { __typename?: 'ActionNetworkSource', apiKey: string, groupSlug: string } | { __typename?: 'AirtableSource', apiKey: string, baseId: string, tableId: string } | { __typename?: 'EditableGoogleSheetsSource' } | { __typename?: 'MailchimpSource', apiKey: string, listId: string } | { __typename?: 'TicketTailorSource', apiKey: string }, lastImportJob?: { __typename?: 'QueueJob', id: string, lastEventAt: any, status: ProcrastinateJobStatus } | null, lastUpdateJob?: { __typename?: 'QueueJob', id: string, lastEventAt: any, status: ProcrastinateJobStatus } | null, importProgress?: { __typename?: 'BatchJobProgress', id: string, hasForecast: boolean, status: ProcrastinateJobStatus, total?: number | null, succeeded?: number | null, estimatedFinishTime?: any | null, actualFinishTime?: any | null, inQueue: boolean, numberOfJobsAheadInQueue?: number | null, sendEmail: boolean } | null, updateProgress?: { __typename?: 'BatchJobProgress', id: string, hasForecast: boolean, status: ProcrastinateJobStatus, total?: number | null, succeeded?: number | null, estimatedFinishTime?: any | null, actualFinishTime?: any | null, inQueue: boolean, numberOfJobsAheadInQueue?: number | null, sendEmail: boolean } | null, fieldDefinitions?: Array<{ __typename?: 'FieldDefinition', label?: string | null, value: string, description?: string | null, editable: boolean }> | null, updateMapping?: Array<{ __typename?: 'AutoUpdateConfig', source: string, sourcePath: string, destinationColumn: string }> | null, sharingPermissions: Array<{ __typename?: 'SharingPermission', id: any }>, organisation: { __typename?: 'Organisation', id: string, name: string } } }; +export type ExternalDataSourceInspectPageQuery = { __typename?: 'Query', externalDataSource: { __typename?: 'ExternalDataSource', id: any, name: string, dataType: DataSourceType, remoteUrl?: string | null, crmType: CrmType, autoImportEnabled: boolean, autoUpdateEnabled: boolean, hasWebhooks: boolean, allowUpdates: boolean, automatedWebhooks: boolean, webhookUrl: string, webhookHealthcheck: boolean, geographyColumn?: string | null, geographyColumnType: GeographyTypes, geocodingConfig: any, usesValidGeocodingConfig: boolean, postcodeField?: string | null, firstNameField?: string | null, lastNameField?: string | null, fullNameField?: string | null, emailField?: string | null, phoneField?: string | null, addressField?: string | null, titleField?: string | null, descriptionField?: string | null, imageField?: string | null, startTimeField?: string | null, endTimeField?: string | null, publicUrlField?: string | null, socialUrlField?: string | null, canDisplayPointField?: string | null, isImportScheduled: boolean, isUpdateScheduled: boolean, importedDataCount: number, importedDataGeocodingRate: number, regionCount: number, constituencyCount: number, ladCount: number, wardCount: number, connectionDetails: { __typename?: 'ActionNetworkSource', apiKey: string, groupSlug: string } | { __typename?: 'AirtableSource', apiKey: string, baseId: string, tableId: string } | { __typename?: 'EditableGoogleSheetsSource' } | { __typename?: 'MailchimpSource', apiKey: string, listId: string } | { __typename?: 'TicketTailorSource', apiKey: string }, lastImportJob?: { __typename?: 'QueueJob', id: string, lastEventAt: any, status: ProcrastinateJobStatus } | null, lastUpdateJob?: { __typename?: 'QueueJob', id: string, lastEventAt: any, status: ProcrastinateJobStatus } | null, importProgress?: { __typename?: 'BatchJobProgress', id: string, hasForecast: boolean, status: ProcrastinateJobStatus, total?: number | null, succeeded?: number | null, estimatedFinishTime?: any | null, actualFinishTime?: any | null, inQueue: boolean, numberOfJobsAheadInQueue?: number | null, sendEmail: boolean } | null, updateProgress?: { __typename?: 'BatchJobProgress', id: string, hasForecast: boolean, status: ProcrastinateJobStatus, total?: number | null, succeeded?: number | null, estimatedFinishTime?: any | null, actualFinishTime?: any | null, inQueue: boolean, numberOfJobsAheadInQueue?: number | null, sendEmail: boolean } | null, fieldDefinitions?: Array<{ __typename?: 'FieldDefinition', label?: string | null, value: string, description?: string | null, editable: boolean }> | null, updateMapping?: Array<{ __typename?: 'AutoUpdateConfig', source: string, sourcePath: string, destinationColumn: string }> | null, sharingPermissions: Array<{ __typename?: 'SharingPermission', id: any }>, organisation: { __typename?: 'Organisation', id: string, name: string } } }; export type DeleteUpdateConfigMutationVariables = Exact<{ id: Scalars['String']['input']; @@ -3075,6 +3237,13 @@ export type DeleteUpdateConfigMutationVariables = Exact<{ export type DeleteUpdateConfigMutation = { __typename?: 'Mutation', deleteExternalDataSource: { __typename?: 'ExternalDataSource', id: any } }; +export type DeleteRecordsMutationVariables = Exact<{ + externalDataSourceId: Scalars['String']['input']; +}>; + + +export type DeleteRecordsMutation = { __typename?: 'Mutation', deleteAllRecords: { __typename?: 'ExternalDataSource', id: any } }; + export type ManageSourceSharingQueryVariables = Exact<{ externalDataSourceId: Scalars['ID']['input']; }>; @@ -3103,6 +3272,14 @@ export type ImportDataMutationVariables = Exact<{ export type ImportDataMutation = { __typename?: 'Mutation', importAll: { __typename?: 'ExternalDataSourceAction', id: string, externalDataSource: { __typename?: 'ExternalDataSource', importedDataCount: number, importProgress?: { __typename?: 'BatchJobProgress', status: ProcrastinateJobStatus, hasForecast: boolean, id: string, total?: number | null, succeeded?: number | null, failed?: number | null, estimatedFinishTime?: any | null, inQueue: boolean } | null } } }; +export type CancelImportMutationVariables = Exact<{ + id: Scalars['String']['input']; + requestId: Scalars['String']['input']; +}>; + + +export type CancelImportMutation = { __typename?: 'Mutation', cancelImport: { __typename?: 'ExternalDataSourceAction', id: string } }; + export type ExternalDataSourceNameQueryVariables = Exact<{ externalDataSourceId: Scalars['ID']['input']; }>; @@ -3458,7 +3635,7 @@ export type UpdateExternalDataSourceMutationVariables = Exact<{ }>; -export type UpdateExternalDataSourceMutation = { __typename?: 'Mutation', updateExternalDataSource: { __typename?: 'ExternalDataSource', id: any, name: string, geographyColumn?: string | null, geographyColumnType: GeographyTypes, postcodeField?: string | null, firstNameField?: string | null, lastNameField?: string | null, emailField?: string | null, phoneField?: string | null, addressField?: string | null, canDisplayPointField?: string | null, autoImportEnabled: boolean, autoUpdateEnabled: boolean, updateMapping?: Array<{ __typename?: 'AutoUpdateConfig', source: string, sourcePath: string, destinationColumn: string }> | null } }; +export type UpdateExternalDataSourceMutation = { __typename?: 'Mutation', updateExternalDataSource: { __typename?: 'ExternalDataSource', id: any, name: string, geographyColumn?: string | null, geographyColumnType: GeographyTypes, geocodingConfig: any, usesValidGeocodingConfig: boolean, postcodeField?: string | null, firstNameField?: string | null, lastNameField?: string | null, emailField?: string | null, phoneField?: string | null, addressField?: string | null, canDisplayPointField?: string | null, autoImportEnabled: boolean, autoUpdateEnabled: boolean, updateMapping?: Array<{ __typename?: 'AutoUpdateConfig', source: string, sourcePath: string, destinationColumn: string }> | null } }; export type MapReportLayersSummaryFragment = { __typename?: 'MapReport', layers: Array<{ __typename?: 'MapLayer', id: string, name: string, sharingPermission?: { __typename?: 'SharingPermission', visibilityRecordDetails?: boolean | null, visibilityRecordCoordinates?: boolean | null, organisation: { __typename?: 'PublicOrganisation', name: string } } | null, source: { __typename?: 'SharedDataSource', id: any, name: string, isImportScheduled: boolean, importedDataCount: number, crmType: CrmType, dataType: DataSourceType, organisation: { __typename?: 'PublicOrganisation', name: string } } }> }; @@ -3483,18 +3660,20 @@ export const LoginDocument = {"kind":"Document","definitions":[{"kind":"Operatio export const PerformPasswordResetDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"PerformPasswordReset"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"token"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"password1"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"password2"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"performPasswordReset"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"token"},"value":{"kind":"Variable","name":{"kind":"Name","value":"token"}}},{"kind":"Argument","name":{"kind":"Name","value":"newPassword1"},"value":{"kind":"Variable","name":{"kind":"Name","value":"password1"}}},{"kind":"Argument","name":{"kind":"Name","value":"newPassword2"},"value":{"kind":"Variable","name":{"kind":"Name","value":"password2"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"errors"}},{"kind":"Field","name":{"kind":"Name","value":"success"}}]}}]}}]} as unknown as DocumentNode; export const ResetPasswordDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"ResetPassword"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"email"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"requestPasswordReset"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"email"},"value":{"kind":"Variable","name":{"kind":"Name","value":"email"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"errors"}},{"kind":"Field","name":{"kind":"Name","value":"success"}}]}}]}}]} as unknown as DocumentNode; export const ListOrganisationsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"ListOrganisations"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"currentOrganisationId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"myOrganisations"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"filters"},"value":{"kind":"ObjectValue","fields":[{"kind":"ObjectField","name":{"kind":"Name","value":"id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"currentOrganisationId"}}}]}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"externalDataSources"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"dataType"}},{"kind":"Field","name":{"kind":"Name","value":"connectionDetails"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"AirtableSource"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"baseId"}},{"kind":"Field","name":{"kind":"Name","value":"tableId"}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"MailchimpSource"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"apiKey"}},{"kind":"Field","name":{"kind":"Name","value":"listId"}}]}}]}},{"kind":"Field","name":{"kind":"Name","value":"crmType"}},{"kind":"Field","name":{"kind":"Name","value":"autoImportEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"autoUpdateEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"jobs"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"pagination"},"value":{"kind":"ObjectValue","fields":[{"kind":"ObjectField","name":{"kind":"Name","value":"limit"},"value":{"kind":"IntValue","value":"10"}}]}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"lastEventAt"}},{"kind":"Field","name":{"kind":"Name","value":"status"}}]}},{"kind":"Field","name":{"kind":"Name","value":"updateMapping"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"source"}},{"kind":"Field","name":{"kind":"Name","value":"sourcePath"}},{"kind":"Field","name":{"kind":"Name","value":"destinationColumn"}}]}},{"kind":"Field","name":{"kind":"Name","value":"sharingPermissions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"organisation"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}}]}}]}},{"kind":"Field","name":{"kind":"Name","value":"sharingPermissionsFromOtherOrgs"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"externalDataSource"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"dataType"}},{"kind":"Field","name":{"kind":"Name","value":"crmType"}},{"kind":"Field","name":{"kind":"Name","value":"organisation"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"name"}}]}}]}}]}}]}}]}}]} as unknown as DocumentNode; -export const GetSourceMappingDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetSourceMapping"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"ID"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"externalDataSource"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"pk"},"value":{"kind":"Variable","name":{"kind":"Name","value":"ID"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"autoImportEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"autoUpdateEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"allowUpdates"}},{"kind":"Field","name":{"kind":"Name","value":"hasWebhooks"}},{"kind":"Field","name":{"kind":"Name","value":"updateMapping"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"destinationColumn"}},{"kind":"Field","name":{"kind":"Name","value":"source"}},{"kind":"Field","name":{"kind":"Name","value":"sourcePath"}}]}},{"kind":"Field","name":{"kind":"Name","value":"fieldDefinitions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"label"}},{"kind":"Field","name":{"kind":"Name","value":"value"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"editable"}}]}},{"kind":"Field","name":{"kind":"Name","value":"crmType"}},{"kind":"Field","name":{"kind":"Name","value":"geographyColumn"}},{"kind":"Field","name":{"kind":"Name","value":"geographyColumnType"}},{"kind":"Field","name":{"kind":"Name","value":"postcodeField"}},{"kind":"Field","name":{"kind":"Name","value":"firstNameField"}},{"kind":"Field","name":{"kind":"Name","value":"lastNameField"}},{"kind":"Field","name":{"kind":"Name","value":"emailField"}},{"kind":"Field","name":{"kind":"Name","value":"phoneField"}},{"kind":"Field","name":{"kind":"Name","value":"addressField"}},{"kind":"Field","name":{"kind":"Name","value":"canDisplayPointField"}}]}}]}}]} as unknown as DocumentNode; -export const TestDataSourceDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"TestDataSource"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"CreateExternalDataSourceInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"testDataSource"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"__typename"}},{"kind":"Field","name":{"kind":"Name","value":"crmType"}},{"kind":"Field","name":{"kind":"Name","value":"fieldDefinitions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"label"}},{"kind":"Field","name":{"kind":"Name","value":"value"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"editable"}}]}},{"kind":"Field","name":{"kind":"Name","value":"geographyColumn"}},{"kind":"Field","name":{"kind":"Name","value":"geographyColumnType"}},{"kind":"Field","name":{"kind":"Name","value":"healthcheck"}},{"kind":"Field","name":{"kind":"Name","value":"predefinedColumnNames"}},{"kind":"Field","name":{"kind":"Name","value":"defaultDataType"}},{"kind":"Field","name":{"kind":"Name","value":"remoteName"}},{"kind":"Field","name":{"kind":"Name","value":"allowUpdates"}},{"kind":"Field","name":{"kind":"Name","value":"defaults"}},{"kind":"Field","name":{"kind":"Name","value":"oauthCredentials"}}]}}]}}]} as unknown as DocumentNode; +export const GetSourceMappingDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetSourceMapping"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"ID"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"externalDataSource"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"pk"},"value":{"kind":"Variable","name":{"kind":"Name","value":"ID"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"autoImportEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"autoUpdateEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"allowUpdates"}},{"kind":"Field","name":{"kind":"Name","value":"hasWebhooks"}},{"kind":"Field","name":{"kind":"Name","value":"updateMapping"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"destinationColumn"}},{"kind":"Field","name":{"kind":"Name","value":"source"}},{"kind":"Field","name":{"kind":"Name","value":"sourcePath"}}]}},{"kind":"Field","name":{"kind":"Name","value":"fieldDefinitions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"label"}},{"kind":"Field","name":{"kind":"Name","value":"value"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"editable"}}]}},{"kind":"Field","name":{"kind":"Name","value":"crmType"}},{"kind":"Field","name":{"kind":"Name","value":"geographyColumn"}},{"kind":"Field","name":{"kind":"Name","value":"geographyColumnType"}},{"kind":"Field","name":{"kind":"Name","value":"geocodingConfig"}},{"kind":"Field","name":{"kind":"Name","value":"usesValidGeocodingConfig"}},{"kind":"Field","name":{"kind":"Name","value":"postcodeField"}},{"kind":"Field","name":{"kind":"Name","value":"firstNameField"}},{"kind":"Field","name":{"kind":"Name","value":"lastNameField"}},{"kind":"Field","name":{"kind":"Name","value":"emailField"}},{"kind":"Field","name":{"kind":"Name","value":"phoneField"}},{"kind":"Field","name":{"kind":"Name","value":"addressField"}},{"kind":"Field","name":{"kind":"Name","value":"canDisplayPointField"}}]}}]}}]} as unknown as DocumentNode; +export const TestDataSourceDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"TestDataSource"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"CreateExternalDataSourceInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"testDataSource"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"__typename"}},{"kind":"Field","name":{"kind":"Name","value":"crmType"}},{"kind":"Field","name":{"kind":"Name","value":"fieldDefinitions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"label"}},{"kind":"Field","name":{"kind":"Name","value":"value"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"editable"}}]}},{"kind":"Field","name":{"kind":"Name","value":"geographyColumn"}},{"kind":"Field","name":{"kind":"Name","value":"geographyColumnType"}},{"kind":"Field","name":{"kind":"Name","value":"geocodingConfig"}},{"kind":"Field","name":{"kind":"Name","value":"usesValidGeocodingConfig"}},{"kind":"Field","name":{"kind":"Name","value":"healthcheck"}},{"kind":"Field","name":{"kind":"Name","value":"predefinedColumnNames"}},{"kind":"Field","name":{"kind":"Name","value":"defaultDataType"}},{"kind":"Field","name":{"kind":"Name","value":"remoteName"}},{"kind":"Field","name":{"kind":"Name","value":"allowUpdates"}},{"kind":"Field","name":{"kind":"Name","value":"defaults"}},{"kind":"Field","name":{"kind":"Name","value":"oauthCredentials"}}]}}]}}]} as unknown as DocumentNode; export const GoogleSheetsOauthUrlDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GoogleSheetsOauthUrl"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"redirectUrl"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"googleSheetsOauthUrl"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"redirectUrl"},"value":{"kind":"Variable","name":{"kind":"Name","value":"redirectUrl"}}}]}]}}]} as unknown as DocumentNode; export const GoogleSheetsOauthCredentialsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GoogleSheetsOauthCredentials"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"redirectSuccessUrl"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"googleSheetsOauthCredentials"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"redirectSuccessUrl"},"value":{"kind":"Variable","name":{"kind":"Name","value":"redirectSuccessUrl"}}}]}]}}]} as unknown as DocumentNode; export const CreateSourceDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"CreateSource"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"CreateExternalDataSourceInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"createExternalDataSource"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"code"}},{"kind":"Field","name":{"kind":"Name","value":"errors"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"message"}}]}},{"kind":"Field","name":{"kind":"Name","value":"result"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"crmType"}},{"kind":"Field","name":{"kind":"Name","value":"dataType"}},{"kind":"Field","name":{"kind":"Name","value":"allowUpdates"}}]}}]}}]}}]} as unknown as DocumentNode; -export const AutoUpdateCreationReviewDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"AutoUpdateCreationReview"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"ID"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"externalDataSource"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"pk"},"value":{"kind":"Variable","name":{"kind":"Name","value":"ID"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"geographyColumn"}},{"kind":"Field","name":{"kind":"Name","value":"geographyColumnType"}},{"kind":"Field","name":{"kind":"Name","value":"dataType"}},{"kind":"Field","name":{"kind":"Name","value":"crmType"}},{"kind":"Field","name":{"kind":"Name","value":"autoImportEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"autoUpdateEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"updateMapping"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"source"}},{"kind":"Field","name":{"kind":"Name","value":"sourcePath"}},{"kind":"Field","name":{"kind":"Name","value":"destinationColumn"}}]}},{"kind":"Field","name":{"kind":"Name","value":"jobs"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"pagination"},"value":{"kind":"ObjectValue","fields":[{"kind":"ObjectField","name":{"kind":"Name","value":"limit"},"value":{"kind":"IntValue","value":"10"}}]}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"lastEventAt"}},{"kind":"Field","name":{"kind":"Name","value":"status"}}]}},{"kind":"Field","name":{"kind":"Name","value":"automatedWebhooks"}},{"kind":"Field","name":{"kind":"Name","value":"webhookUrl"}},{"kind":"FragmentSpread","name":{"kind":"Name","value":"DataSourceCard"}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"DataSourceCard"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ExternalDataSource"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"dataType"}},{"kind":"Field","name":{"kind":"Name","value":"crmType"}},{"kind":"Field","name":{"kind":"Name","value":"automatedWebhooks"}},{"kind":"Field","name":{"kind":"Name","value":"autoImportEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"autoUpdateEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"updateMapping"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"source"}},{"kind":"Field","name":{"kind":"Name","value":"sourcePath"}},{"kind":"Field","name":{"kind":"Name","value":"destinationColumn"}}]}},{"kind":"Field","name":{"kind":"Name","value":"jobs"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"pagination"},"value":{"kind":"ObjectValue","fields":[{"kind":"ObjectField","name":{"kind":"Name","value":"limit"},"value":{"kind":"IntValue","value":"10"}}]}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"lastEventAt"}},{"kind":"Field","name":{"kind":"Name","value":"status"}}]}},{"kind":"Field","name":{"kind":"Name","value":"sharingPermissions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"organisation"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}}]}}]}}]} as unknown as DocumentNode; -export const ExternalDataSourceInspectPageDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"ExternalDataSourceInspectPage"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"ID"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"externalDataSource"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"pk"},"value":{"kind":"Variable","name":{"kind":"Name","value":"ID"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"dataType"}},{"kind":"Field","name":{"kind":"Name","value":"remoteUrl"}},{"kind":"Field","name":{"kind":"Name","value":"crmType"}},{"kind":"Field","name":{"kind":"Name","value":"connectionDetails"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"AirtableSource"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"apiKey"}},{"kind":"Field","name":{"kind":"Name","value":"baseId"}},{"kind":"Field","name":{"kind":"Name","value":"tableId"}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"MailchimpSource"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"apiKey"}},{"kind":"Field","name":{"kind":"Name","value":"listId"}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ActionNetworkSource"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"apiKey"}},{"kind":"Field","name":{"kind":"Name","value":"groupSlug"}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"TicketTailorSource"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"apiKey"}}]}}]}},{"kind":"Field","name":{"kind":"Name","value":"lastImportJob"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"lastEventAt"}},{"kind":"Field","name":{"kind":"Name","value":"status"}}]}},{"kind":"Field","name":{"kind":"Name","value":"lastUpdateJob"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"lastEventAt"}},{"kind":"Field","name":{"kind":"Name","value":"status"}}]}},{"kind":"Field","name":{"kind":"Name","value":"autoImportEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"autoUpdateEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"hasWebhooks"}},{"kind":"Field","name":{"kind":"Name","value":"allowUpdates"}},{"kind":"Field","name":{"kind":"Name","value":"automatedWebhooks"}},{"kind":"Field","name":{"kind":"Name","value":"webhookUrl"}},{"kind":"Field","name":{"kind":"Name","value":"webhookHealthcheck"}},{"kind":"Field","name":{"kind":"Name","value":"geographyColumn"}},{"kind":"Field","name":{"kind":"Name","value":"geographyColumnType"}},{"kind":"Field","name":{"kind":"Name","value":"postcodeField"}},{"kind":"Field","name":{"kind":"Name","value":"firstNameField"}},{"kind":"Field","name":{"kind":"Name","value":"lastNameField"}},{"kind":"Field","name":{"kind":"Name","value":"fullNameField"}},{"kind":"Field","name":{"kind":"Name","value":"emailField"}},{"kind":"Field","name":{"kind":"Name","value":"phoneField"}},{"kind":"Field","name":{"kind":"Name","value":"addressField"}},{"kind":"Field","name":{"kind":"Name","value":"titleField"}},{"kind":"Field","name":{"kind":"Name","value":"descriptionField"}},{"kind":"Field","name":{"kind":"Name","value":"imageField"}},{"kind":"Field","name":{"kind":"Name","value":"startTimeField"}},{"kind":"Field","name":{"kind":"Name","value":"endTimeField"}},{"kind":"Field","name":{"kind":"Name","value":"publicUrlField"}},{"kind":"Field","name":{"kind":"Name","value":"socialUrlField"}},{"kind":"Field","name":{"kind":"Name","value":"canDisplayPointField"}},{"kind":"Field","name":{"kind":"Name","value":"isImportScheduled"}},{"kind":"Field","name":{"kind":"Name","value":"importProgress"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"hasForecast"}},{"kind":"Field","name":{"kind":"Name","value":"status"}},{"kind":"Field","name":{"kind":"Name","value":"total"}},{"kind":"Field","name":{"kind":"Name","value":"succeeded"}},{"kind":"Field","name":{"kind":"Name","value":"estimatedFinishTime"}},{"kind":"Field","name":{"kind":"Name","value":"actualFinishTime"}},{"kind":"Field","name":{"kind":"Name","value":"inQueue"}},{"kind":"Field","name":{"kind":"Name","value":"numberOfJobsAheadInQueue"}},{"kind":"Field","name":{"kind":"Name","value":"sendEmail"}}]}},{"kind":"Field","name":{"kind":"Name","value":"isUpdateScheduled"}},{"kind":"Field","name":{"kind":"Name","value":"updateProgress"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"hasForecast"}},{"kind":"Field","name":{"kind":"Name","value":"status"}},{"kind":"Field","name":{"kind":"Name","value":"total"}},{"kind":"Field","name":{"kind":"Name","value":"succeeded"}},{"kind":"Field","name":{"kind":"Name","value":"estimatedFinishTime"}},{"kind":"Field","name":{"kind":"Name","value":"actualFinishTime"}},{"kind":"Field","name":{"kind":"Name","value":"inQueue"}},{"kind":"Field","name":{"kind":"Name","value":"numberOfJobsAheadInQueue"}},{"kind":"Field","name":{"kind":"Name","value":"sendEmail"}}]}},{"kind":"Field","name":{"kind":"Name","value":"importedDataCount"}},{"kind":"Field","name":{"kind":"Name","value":"fieldDefinitions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"label"}},{"kind":"Field","name":{"kind":"Name","value":"value"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"editable"}}]}},{"kind":"Field","name":{"kind":"Name","value":"updateMapping"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"source"}},{"kind":"Field","name":{"kind":"Name","value":"sourcePath"}},{"kind":"Field","name":{"kind":"Name","value":"destinationColumn"}}]}},{"kind":"Field","name":{"kind":"Name","value":"sharingPermissions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}}]}},{"kind":"Field","name":{"kind":"Name","value":"organisation"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}}]}}]}}]} as unknown as DocumentNode; +export const AutoUpdateCreationReviewDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"AutoUpdateCreationReview"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"ID"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"externalDataSource"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"pk"},"value":{"kind":"Variable","name":{"kind":"Name","value":"ID"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"geographyColumn"}},{"kind":"Field","name":{"kind":"Name","value":"geographyColumnType"}},{"kind":"Field","name":{"kind":"Name","value":"geocodingConfig"}},{"kind":"Field","name":{"kind":"Name","value":"usesValidGeocodingConfig"}},{"kind":"Field","name":{"kind":"Name","value":"dataType"}},{"kind":"Field","name":{"kind":"Name","value":"crmType"}},{"kind":"Field","name":{"kind":"Name","value":"autoImportEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"autoUpdateEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"updateMapping"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"source"}},{"kind":"Field","name":{"kind":"Name","value":"sourcePath"}},{"kind":"Field","name":{"kind":"Name","value":"destinationColumn"}}]}},{"kind":"Field","name":{"kind":"Name","value":"jobs"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"pagination"},"value":{"kind":"ObjectValue","fields":[{"kind":"ObjectField","name":{"kind":"Name","value":"limit"},"value":{"kind":"IntValue","value":"10"}}]}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"lastEventAt"}},{"kind":"Field","name":{"kind":"Name","value":"status"}}]}},{"kind":"Field","name":{"kind":"Name","value":"automatedWebhooks"}},{"kind":"Field","name":{"kind":"Name","value":"webhookUrl"}},{"kind":"FragmentSpread","name":{"kind":"Name","value":"DataSourceCard"}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"DataSourceCard"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ExternalDataSource"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"dataType"}},{"kind":"Field","name":{"kind":"Name","value":"crmType"}},{"kind":"Field","name":{"kind":"Name","value":"automatedWebhooks"}},{"kind":"Field","name":{"kind":"Name","value":"autoImportEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"autoUpdateEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"updateMapping"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"source"}},{"kind":"Field","name":{"kind":"Name","value":"sourcePath"}},{"kind":"Field","name":{"kind":"Name","value":"destinationColumn"}}]}},{"kind":"Field","name":{"kind":"Name","value":"jobs"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"pagination"},"value":{"kind":"ObjectValue","fields":[{"kind":"ObjectField","name":{"kind":"Name","value":"limit"},"value":{"kind":"IntValue","value":"10"}}]}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"lastEventAt"}},{"kind":"Field","name":{"kind":"Name","value":"status"}}]}},{"kind":"Field","name":{"kind":"Name","value":"sharingPermissions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"organisation"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}}]}}]}}]} as unknown as DocumentNode; +export const ExternalDataSourceInspectPageDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"ExternalDataSourceInspectPage"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"ID"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"externalDataSource"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"pk"},"value":{"kind":"Variable","name":{"kind":"Name","value":"ID"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"dataType"}},{"kind":"Field","name":{"kind":"Name","value":"remoteUrl"}},{"kind":"Field","name":{"kind":"Name","value":"crmType"}},{"kind":"Field","name":{"kind":"Name","value":"connectionDetails"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"AirtableSource"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"apiKey"}},{"kind":"Field","name":{"kind":"Name","value":"baseId"}},{"kind":"Field","name":{"kind":"Name","value":"tableId"}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"MailchimpSource"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"apiKey"}},{"kind":"Field","name":{"kind":"Name","value":"listId"}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ActionNetworkSource"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"apiKey"}},{"kind":"Field","name":{"kind":"Name","value":"groupSlug"}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"TicketTailorSource"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"apiKey"}}]}}]}},{"kind":"Field","name":{"kind":"Name","value":"lastImportJob"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"lastEventAt"}},{"kind":"Field","name":{"kind":"Name","value":"status"}}]}},{"kind":"Field","name":{"kind":"Name","value":"lastUpdateJob"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"lastEventAt"}},{"kind":"Field","name":{"kind":"Name","value":"status"}}]}},{"kind":"Field","name":{"kind":"Name","value":"autoImportEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"autoUpdateEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"hasWebhooks"}},{"kind":"Field","name":{"kind":"Name","value":"allowUpdates"}},{"kind":"Field","name":{"kind":"Name","value":"automatedWebhooks"}},{"kind":"Field","name":{"kind":"Name","value":"webhookUrl"}},{"kind":"Field","name":{"kind":"Name","value":"webhookHealthcheck"}},{"kind":"Field","name":{"kind":"Name","value":"geographyColumn"}},{"kind":"Field","name":{"kind":"Name","value":"geographyColumnType"}},{"kind":"Field","name":{"kind":"Name","value":"geocodingConfig"}},{"kind":"Field","name":{"kind":"Name","value":"usesValidGeocodingConfig"}},{"kind":"Field","name":{"kind":"Name","value":"postcodeField"}},{"kind":"Field","name":{"kind":"Name","value":"firstNameField"}},{"kind":"Field","name":{"kind":"Name","value":"lastNameField"}},{"kind":"Field","name":{"kind":"Name","value":"fullNameField"}},{"kind":"Field","name":{"kind":"Name","value":"emailField"}},{"kind":"Field","name":{"kind":"Name","value":"phoneField"}},{"kind":"Field","name":{"kind":"Name","value":"addressField"}},{"kind":"Field","name":{"kind":"Name","value":"titleField"}},{"kind":"Field","name":{"kind":"Name","value":"descriptionField"}},{"kind":"Field","name":{"kind":"Name","value":"imageField"}},{"kind":"Field","name":{"kind":"Name","value":"startTimeField"}},{"kind":"Field","name":{"kind":"Name","value":"endTimeField"}},{"kind":"Field","name":{"kind":"Name","value":"publicUrlField"}},{"kind":"Field","name":{"kind":"Name","value":"socialUrlField"}},{"kind":"Field","name":{"kind":"Name","value":"canDisplayPointField"}},{"kind":"Field","name":{"kind":"Name","value":"isImportScheduled"}},{"kind":"Field","name":{"kind":"Name","value":"importProgress"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"hasForecast"}},{"kind":"Field","name":{"kind":"Name","value":"status"}},{"kind":"Field","name":{"kind":"Name","value":"total"}},{"kind":"Field","name":{"kind":"Name","value":"succeeded"}},{"kind":"Field","name":{"kind":"Name","value":"estimatedFinishTime"}},{"kind":"Field","name":{"kind":"Name","value":"actualFinishTime"}},{"kind":"Field","name":{"kind":"Name","value":"inQueue"}},{"kind":"Field","name":{"kind":"Name","value":"numberOfJobsAheadInQueue"}},{"kind":"Field","name":{"kind":"Name","value":"sendEmail"}}]}},{"kind":"Field","name":{"kind":"Name","value":"isUpdateScheduled"}},{"kind":"Field","name":{"kind":"Name","value":"updateProgress"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"hasForecast"}},{"kind":"Field","name":{"kind":"Name","value":"status"}},{"kind":"Field","name":{"kind":"Name","value":"total"}},{"kind":"Field","name":{"kind":"Name","value":"succeeded"}},{"kind":"Field","name":{"kind":"Name","value":"estimatedFinishTime"}},{"kind":"Field","name":{"kind":"Name","value":"actualFinishTime"}},{"kind":"Field","name":{"kind":"Name","value":"inQueue"}},{"kind":"Field","name":{"kind":"Name","value":"numberOfJobsAheadInQueue"}},{"kind":"Field","name":{"kind":"Name","value":"sendEmail"}}]}},{"kind":"Field","name":{"kind":"Name","value":"importedDataCount"}},{"kind":"Field","name":{"kind":"Name","value":"importedDataGeocodingRate"}},{"kind":"Field","alias":{"kind":"Name","value":"regionCount"},"name":{"kind":"Name","value":"importedDataCountOfAreas"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"analyticalAreaType"},"value":{"kind":"EnumValue","value":"european_electoral_region"}}]},{"kind":"Field","alias":{"kind":"Name","value":"constituencyCount"},"name":{"kind":"Name","value":"importedDataCountOfAreas"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"analyticalAreaType"},"value":{"kind":"EnumValue","value":"parliamentary_constituency"}}]},{"kind":"Field","alias":{"kind":"Name","value":"ladCount"},"name":{"kind":"Name","value":"importedDataCountOfAreas"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"analyticalAreaType"},"value":{"kind":"EnumValue","value":"admin_district"}}]},{"kind":"Field","alias":{"kind":"Name","value":"wardCount"},"name":{"kind":"Name","value":"importedDataCountOfAreas"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"analyticalAreaType"},"value":{"kind":"EnumValue","value":"admin_ward"}}]},{"kind":"Field","name":{"kind":"Name","value":"fieldDefinitions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"label"}},{"kind":"Field","name":{"kind":"Name","value":"value"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"editable"}}]}},{"kind":"Field","name":{"kind":"Name","value":"updateMapping"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"source"}},{"kind":"Field","name":{"kind":"Name","value":"sourcePath"}},{"kind":"Field","name":{"kind":"Name","value":"destinationColumn"}}]}},{"kind":"Field","name":{"kind":"Name","value":"sharingPermissions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}}]}},{"kind":"Field","name":{"kind":"Name","value":"organisation"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}}]}}]}}]} as unknown as DocumentNode; export const DeleteUpdateConfigDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"DeleteUpdateConfig"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"id"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"deleteExternalDataSource"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"data"},"value":{"kind":"ObjectValue","fields":[{"kind":"ObjectField","name":{"kind":"Name","value":"id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"id"}}}]}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}}]}}]}}]} as unknown as DocumentNode; +export const DeleteRecordsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"DeleteRecords"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"externalDataSourceId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"deleteAllRecords"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"externalDataSourceId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"externalDataSourceId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}}]}}]}}]} as unknown as DocumentNode; export const ManageSourceSharingDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"ManageSourceSharing"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"externalDataSourceId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"externalDataSource"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"pk"},"value":{"kind":"Variable","name":{"kind":"Name","value":"externalDataSourceId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"sharingPermissions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"organisationId"}},{"kind":"Field","name":{"kind":"Name","value":"organisation"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"name"}}]}},{"kind":"Field","name":{"kind":"Name","value":"externalDataSourceId"}},{"kind":"Field","name":{"kind":"Name","value":"visibilityRecordCoordinates"}},{"kind":"Field","name":{"kind":"Name","value":"visibilityRecordDetails"}},{"kind":"Field","name":{"kind":"Name","value":"deleted"}}]}}]}}]}}]} as unknown as DocumentNode; export const UpdateSourceSharingObjectDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"UpdateSourceSharingObject"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"data"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"SharingPermissionCUDInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"updateSharingPermission"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"data"},"value":{"kind":"Variable","name":{"kind":"Name","value":"data"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"organisationId"}},{"kind":"Field","name":{"kind":"Name","value":"externalDataSourceId"}},{"kind":"Field","name":{"kind":"Name","value":"visibilityRecordCoordinates"}},{"kind":"Field","name":{"kind":"Name","value":"visibilityRecordDetails"}},{"kind":"Field","name":{"kind":"Name","value":"deleted"}}]}}]}}]} as unknown as DocumentNode; export const DeleteSourceSharingObjectDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"DeleteSourceSharingObject"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"pk"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"deleteSharingPermission"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"data"},"value":{"kind":"ObjectValue","fields":[{"kind":"ObjectField","name":{"kind":"Name","value":"id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"pk"}}}]}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}}]}}]}}]} as unknown as DocumentNode; export const ImportDataDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"ImportData"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"id"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"importAll"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"externalDataSourceId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"id"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"externalDataSource"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"importedDataCount"}},{"kind":"Field","name":{"kind":"Name","value":"importProgress"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"status"}},{"kind":"Field","name":{"kind":"Name","value":"hasForecast"}},{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"total"}},{"kind":"Field","name":{"kind":"Name","value":"succeeded"}},{"kind":"Field","name":{"kind":"Name","value":"failed"}},{"kind":"Field","name":{"kind":"Name","value":"estimatedFinishTime"}},{"kind":"Field","name":{"kind":"Name","value":"inQueue"}}]}}]}}]}}]}}]} as unknown as DocumentNode; +export const CancelImportDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"CancelImport"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"id"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"requestId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"cancelImport"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"externalDataSourceId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"id"}}},{"kind":"Argument","name":{"kind":"Name","value":"requestId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"requestId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}}]}}]}}]} as unknown as DocumentNode; export const ExternalDataSourceNameDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"ExternalDataSourceName"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"externalDataSourceId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"externalDataSource"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"pk"},"value":{"kind":"Variable","name":{"kind":"Name","value":"externalDataSourceId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"crmType"}},{"kind":"Field","name":{"kind":"Name","value":"dataType"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"remoteUrl"}}]}}]}}]} as unknown as DocumentNode; export const ShareDataSourcesDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"ShareDataSources"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"fromOrgId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"permissions"}},"type":{"kind":"NonNullType","type":{"kind":"ListType","type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"SharingPermissionInput"}}}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"updateSharingPermissions"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"fromOrgId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"fromOrgId"}}},{"kind":"Argument","name":{"kind":"Name","value":"permissions"},"value":{"kind":"Variable","name":{"kind":"Name","value":"permissions"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"sharingPermissions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"organisationId"}},{"kind":"Field","name":{"kind":"Name","value":"externalDataSourceId"}},{"kind":"Field","name":{"kind":"Name","value":"visibilityRecordCoordinates"}},{"kind":"Field","name":{"kind":"Name","value":"visibilityRecordDetails"}},{"kind":"Field","name":{"kind":"Name","value":"deleted"}}]}}]}}]}}]} as unknown as DocumentNode; export const YourSourcesForSharingDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"YourSourcesForSharing"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"myOrganisations"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"externalDataSources"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"crmType"}},{"kind":"Field","name":{"kind":"Name","value":"importedDataCount"}},{"kind":"Field","name":{"kind":"Name","value":"dataType"}},{"kind":"Field","name":{"kind":"Name","value":"fieldDefinitions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"label"}},{"kind":"Field","name":{"kind":"Name","value":"editable"}}]}},{"kind":"Field","name":{"kind":"Name","value":"organisationId"}},{"kind":"Field","name":{"kind":"Name","value":"sharingPermissions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"organisationId"}},{"kind":"Field","name":{"kind":"Name","value":"externalDataSourceId"}},{"kind":"Field","name":{"kind":"Name","value":"visibilityRecordCoordinates"}},{"kind":"Field","name":{"kind":"Name","value":"visibilityRecordDetails"}},{"kind":"Field","name":{"kind":"Name","value":"deleted"}}]}}]}}]}}]}}]} as unknown as DocumentNode; @@ -3543,7 +3722,7 @@ export const GetEventListDocument = {"kind":"Document","definitions":[{"kind":"O export const GetHubHomepageJsonDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetHubHomepageJson"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"hostname"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"hubPageByPath"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"hostname"},"value":{"kind":"Variable","name":{"kind":"Name","value":"hostname"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"puckJsonContent"}}]}}]}}]} as unknown as DocumentNode; export const HubListDataSourcesDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"HubListDataSources"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"currentOrganisationId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"myOrganisations"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"filters"},"value":{"kind":"ObjectValue","fields":[{"kind":"ObjectField","name":{"kind":"Name","value":"id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"currentOrganisationId"}}}]}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"externalDataSources"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"dataType"}}]}}]}}]}}]} as unknown as DocumentNode; export const AddMemberDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"AddMember"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"externalDataSourceId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"email"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"postcode"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"customFields"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"JSON"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"tags"}},"type":{"kind":"NonNullType","type":{"kind":"ListType","type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"addMember"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"externalDataSourceId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"externalDataSourceId"}}},{"kind":"Argument","name":{"kind":"Name","value":"email"},"value":{"kind":"Variable","name":{"kind":"Name","value":"email"}}},{"kind":"Argument","name":{"kind":"Name","value":"postcode"},"value":{"kind":"Variable","name":{"kind":"Name","value":"postcode"}}},{"kind":"Argument","name":{"kind":"Name","value":"customFields"},"value":{"kind":"Variable","name":{"kind":"Name","value":"customFields"}}},{"kind":"Argument","name":{"kind":"Name","value":"tags"},"value":{"kind":"Variable","name":{"kind":"Name","value":"tags"}}}]}]}}]} as unknown as DocumentNode; -export const UpdateExternalDataSourceDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"UpdateExternalDataSource"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ExternalDataSourceInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"updateExternalDataSource"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"geographyColumn"}},{"kind":"Field","name":{"kind":"Name","value":"geographyColumnType"}},{"kind":"Field","name":{"kind":"Name","value":"postcodeField"}},{"kind":"Field","name":{"kind":"Name","value":"firstNameField"}},{"kind":"Field","name":{"kind":"Name","value":"lastNameField"}},{"kind":"Field","name":{"kind":"Name","value":"emailField"}},{"kind":"Field","name":{"kind":"Name","value":"phoneField"}},{"kind":"Field","name":{"kind":"Name","value":"addressField"}},{"kind":"Field","name":{"kind":"Name","value":"canDisplayPointField"}},{"kind":"Field","name":{"kind":"Name","value":"autoImportEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"autoUpdateEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"updateMapping"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"source"}},{"kind":"Field","name":{"kind":"Name","value":"sourcePath"}},{"kind":"Field","name":{"kind":"Name","value":"destinationColumn"}}]}}]}}]}}]} as unknown as DocumentNode; +export const UpdateExternalDataSourceDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"UpdateExternalDataSource"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ExternalDataSourceInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"updateExternalDataSource"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"geographyColumn"}},{"kind":"Field","name":{"kind":"Name","value":"geographyColumnType"}},{"kind":"Field","name":{"kind":"Name","value":"geocodingConfig"}},{"kind":"Field","name":{"kind":"Name","value":"usesValidGeocodingConfig"}},{"kind":"Field","name":{"kind":"Name","value":"postcodeField"}},{"kind":"Field","name":{"kind":"Name","value":"firstNameField"}},{"kind":"Field","name":{"kind":"Name","value":"lastNameField"}},{"kind":"Field","name":{"kind":"Name","value":"emailField"}},{"kind":"Field","name":{"kind":"Name","value":"phoneField"}},{"kind":"Field","name":{"kind":"Name","value":"addressField"}},{"kind":"Field","name":{"kind":"Name","value":"canDisplayPointField"}},{"kind":"Field","name":{"kind":"Name","value":"autoImportEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"autoUpdateEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"updateMapping"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"source"}},{"kind":"Field","name":{"kind":"Name","value":"sourcePath"}},{"kind":"Field","name":{"kind":"Name","value":"destinationColumn"}}]}}]}}]}}]} as unknown as DocumentNode; export const PublicUserDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"PublicUser"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"publicUser"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"username"}},{"kind":"Field","name":{"kind":"Name","value":"email"}}]}}]}}]} as unknown as DocumentNode; export interface PossibleTypesResultData { diff --git a/nextjs/src/app/(logged-in)/data-sources/create/configure/[externalDataSourceId]/page.tsx b/nextjs/src/app/(logged-in)/data-sources/create/configure/[externalDataSourceId]/page.tsx index 4363735c4..1a454e0a5 100644 --- a/nextjs/src/app/(logged-in)/data-sources/create/configure/[externalDataSourceId]/page.tsx +++ b/nextjs/src/app/(logged-in)/data-sources/create/configure/[externalDataSourceId]/page.tsx @@ -43,6 +43,8 @@ const GET_UPDATE_CONFIG = gql` crmType geographyColumn geographyColumnType + geocodingConfig + usesValidGeocodingConfig postcodeField firstNameField lastNameField @@ -148,6 +150,7 @@ export default function Page({ {externalDataSource.data ? (
@@ -302,6 +315,24 @@ export default function InspectExternalDataSource({
{format(',')(source.importedDataCount || 0)}
+
+ Located in {pluralize('region', source.regionCount, true)},{' '} + {pluralize('constituency', source.constituencyCount, true)},{' '} + {pluralize('local authority', source.ladCount, true)}, and{' '} + {pluralize('ward', source.wardCount, true)}.{' '} + {/* Un-geolocated count: */} + + Geocoding success rate of{' '} + {(source.importedDataGeocodingRate || 0).toFixed(0)}% + +

Import data from this {formatCrmNames(source.crmType)} into Mapped for use in reports @@ -310,24 +341,59 @@ export default function InspectExternalDataSource({ : ''} .

- + + {source.importProgress?.inQueue && ( + )} - + + {source.importProgress?.status !== 'todo' ? ( + {!!source.sharingPermissions?.length && ( @@ -496,14 +569,20 @@ export default function InspectExternalDataSource({

Pull Mapped data into your {crmInfo?.name || 'database'}{' '} - based on each record - {"'"}s - + {!source.geocodingConfig && ( + <> + + based on each record + {"'"}s + + + + )}

{!source.updateProgress?.inQueue ? ( @@ -633,12 +712,14 @@ export default function InspectExternalDataSource({ after changing these settings.

{ refetch() }} + allowGeocodingConfigChange={!source.usesValidGeocodingConfig} + externalDataSourceId={source.id} initialData={{ // Trim out the __typenames geographyColumn: source?.geographyColumn, @@ -683,43 +764,114 @@ export default function InspectExternalDataSource({ {source.connectionDetails.apiKey}
) : null} - - - - - - - Are you sure? - - This action cannot be undone. This will permanently delete - this data source connect from Mapped. - - - - - Cancel - - { - del() - }} - className={buttonVariants({ variant: 'destructive' })} - > - Confirm delete - - - - + +
+ + {/* */} + { + toastPromise( + client.mutate({ + mutation: gql` + mutation DeleteRecords($externalDataSourceId: String!) { + deleteAllRecords( + externalDataSourceId: $externalDataSourceId + ) { + id + } + } + `, + variables: { + externalDataSourceId, + }, + }), + { + loading: 'Deleting all records...', + success: () => { + refetch() + return 'Deleted all records' + }, + error: 'Failed to delete', + } + ) + }} + /> +
)} ) + function UpdateGecodingConfig({ + externalDataSourceId, + geocodingConfig, + fieldDefinitions, + onSubmit, + }: { + externalDataSourceId: string + geocodingConfig: any + fieldDefinitions: FieldDefinition[] | null | undefined + onSubmit: ( + data: ExternalDataSourceInput, + e?: React.BaseSyntheticEvent | undefined + ) => void + }) { + const [newGeocodingConfig, setGeocodingConfig] = useState( + geocodingConfig ? JSON.stringify(geocodingConfig, null, 2) : '' + ) + return ( + + + + + + + + Only play with this if you know what you are doing. +