diff --git a/hub/graphql/mutations.py b/hub/graphql/mutations.py
index 7351b3f54..22f07dd83 100644
--- a/hub/graphql/mutations.py
+++ b/hub/graphql/mutations.py
@@ -10,6 +10,7 @@
import strawberry_django
from asgiref.sync import async_to_sync
from graphql import GraphQLError
+from procrastinate.contrib.django.models import ProcrastinateJob
from strawberry import auto
from strawberry.field_extensions import InputMutationExtension
from strawberry.types.info import Info
@@ -20,6 +21,7 @@
from hub.graphql.types import model_types
from hub.graphql.utils import graphql_type_to_dict
from hub.models import BatchRequest
+from hub.permissions import user_can_manage_source
logger = logging.getLogger(__name__)
@@ -253,6 +255,45 @@ async def import_all(
return ExternalDataSourceAction(id=request_id, external_data_source=data_source)
+@strawberry_django.mutation(extensions=[IsAuthenticated()])
+def cancel_import(
+ info: Info, external_data_source_id: str, request_id: str
+) -> ExternalDataSourceAction:
+ data_source: models.ExternalDataSource = models.ExternalDataSource.objects.get(
+ id=external_data_source_id
+ )
+ # Confirm user has access to this source
+ user = get_current_user(info)
+ assert user_can_manage_source(user, data_source)
+ # Update all remaining procrastinate jobs, cancel them
+ ProcrastinateJob.objects.filter(
+ args__external_data_source_id=external_data_source_id,
+ status__in=["todo", "doing"],
+ args__request_id=request_id,
+ ).update(status="cancelled")
+ BatchRequest.objects.filter(id=request_id).update(status="cancelled")
+ #
+ return ExternalDataSourceAction(id=request_id, external_data_source=data_source)
+
+
+@strawberry_django.mutation(extensions=[IsAuthenticated()])
+def delete_all_records(
+ info: Info, external_data_source_id: str
+) -> model_types.ExternalDataSource:
+ data_source = models.ExternalDataSource.objects.get(id=external_data_source_id)
+ # Confirm user has access to this source
+ user = get_current_user(info)
+ assert user_can_manage_source(user, data_source)
+ # Don't import more records, since we want to wipe 'em
+ ProcrastinateJob.objects.filter(
+ args__external_data_source_id=external_data_source_id,
+ status__in=["todo", "doing"],
+ ).update(status="cancelled")
+ # Delete all data
+ data_source.get_import_data().all().delete()
+ return models.ExternalDataSource.objects.get(id=external_data_source_id)
+
+
@strawberry_django.input(models.ExternalDataSource, partial=True)
class ExternalDataSourceInput:
id: auto
diff --git a/hub/graphql/schema.py b/hub/graphql/schema.py
index 39cdb778d..a057bd586 100644
--- a/hub/graphql/schema.py
+++ b/hub/graphql/schema.py
@@ -181,6 +181,12 @@ class Mutation:
)
import_all: mutation_types.ExternalDataSourceAction = mutation_types.import_all
+ cancel_import: mutation_types.ExternalDataSourceAction = (
+ mutation_types.cancel_import
+ )
+ delete_all_records: model_types.ExternalDataSource = (
+ mutation_types.delete_all_records
+ )
create_map_report: model_types.MapReport = mutation_types.create_map_report
update_map_report: model_types.MapReport = django_mutations.update(
diff --git a/hub/graphql/types/model_types.py b/hub/graphql/types/model_types.py
index dcdd75de4..4e8ebe954 100644
--- a/hub/graphql/types/model_types.py
+++ b/hub/graphql/types/model_types.py
@@ -49,6 +49,7 @@ class ProcrastinateJobStatus(Enum):
doing = "doing" #: A worker is running the job
succeeded = "succeeded" #: The job ended successfully
failed = "failed" #: The job ended with an error
+ cancelled = "cancelled" #: The job was cancelled
@strawberry_django.filters.filter(
diff --git a/hub/models.py b/hub/models.py
index adfa7a31c..7167c863d 100644
--- a/hub/models.py
+++ b/hub/models.py
@@ -1337,12 +1337,20 @@ def get_scheduled_batch_job_progress(self, parent_job: ProcrastinateJob, user=No
jobs = self.event_log_queryset().filter(args__request_id=request_id).all()
status = "todo"
+ number_of_jobs_ahead_in_queue = (
+ ProcrastinateJob.objects.filter(id__lt=parent_job.id)
+ .filter(status__in=["todo", "doing"])
+ .count()
+ )
+
if any([job.status == "doing" for job in jobs]):
status = "doing"
elif any([job.status == "failed" for job in jobs]):
status = "failed"
elif all([job.status == "succeeded" for job in jobs]):
status = "succeeded"
+ elif number_of_jobs_ahead_in_queue <= 0:
+ status = "succeeded"
total = 0
statuses = dict()
@@ -1361,12 +1369,6 @@ def get_scheduled_batch_job_progress(self, parent_job: ProcrastinateJob, user=No
+ statuses.get("doing", 0)
)
- number_of_jobs_ahead_in_queue = (
- ProcrastinateJob.objects.filter(id__lt=parent_job.id)
- .filter(status__in=["todo", "doing"])
- .count()
- )
-
time_started = (
ProcrastinateEvent.objects.filter(job_id=parent_job.id)
.order_by("at")
@@ -1839,7 +1841,7 @@ async def update_many(self, mapped_records: list[MappedMember], **kwargs):
"Update many not implemented for this data source type."
)
- def get_record_id(self, record):
+ def get_record_id(self, record) -> Optional[Union[str, int]]:
"""
Get the ID for a record.
"""
@@ -2691,7 +2693,7 @@ def field_definitions(self):
]
def get_record_id(self, record: dict):
- return record[self.id_field]
+ return record.get(self.id_field, None)
async def fetch_one(self, member_id):
return self.df[self.df[self.id_field] == member_id].to_dict(orient="records")[0]
@@ -2833,7 +2835,7 @@ async def fetch_all(self):
return itertools.chain.from_iterable(self.table.iterate())
def get_record_id(self, record):
- return record["id"]
+ return record.get("id", None)
def get_record_field(self, record, field, field_type=None):
d = record["fields"].get(str(field), None)
@@ -3122,7 +3124,7 @@ def healthcheck(self):
return False
def get_record_id(self, record):
- return record["id"]
+ return record.get("id", None)
def get_record_field(self, record, field: str, field_type=None):
field_options = [
diff --git a/hub/permissions.py b/hub/permissions.py
new file mode 100644
index 000000000..64a489f4f
--- /dev/null
+++ b/hub/permissions.py
@@ -0,0 +1,2 @@
+def user_can_manage_source(user, source):
+ return source.organisation.members.filter(user=user).exists()
diff --git a/hub/tests/fixtures/geocoding_cases.py b/hub/tests/fixtures/geocoding_cases.py
index 550696164..cadac5a97 100644
--- a/hub/tests/fixtures/geocoding_cases.py
+++ b/hub/tests/fixtures/geocoding_cases.py
@@ -284,4 +284,12 @@
"expected_area_type_code": "WD23",
"expected_area_gss": "E05014252",
},
+ # Failed again
+ {
+ "id": "West LancashireBurscough Bridge & Rufford",
+ "ward": "Burscough Bridge & Rufford",
+ "council": "West Lancashire",
+ "expected_area_type_code": "WD23",
+ "expected_area_gss": "E05014930",
+ },
]
diff --git a/nextjs/src/__generated__/gql.ts b/nextjs/src/__generated__/gql.ts
index 3e393d409..c8e92855f 100644
--- a/nextjs/src/__generated__/gql.ts
+++ b/nextjs/src/__generated__/gql.ts
@@ -31,10 +31,12 @@ const documents = {
"\n query AutoUpdateCreationReview($ID: ID!) {\n externalDataSource(pk: $ID) {\n id\n name\n geographyColumn\n geographyColumnType\n geocodingConfig\n usesValidGeocodingConfig\n dataType\n crmType\n autoImportEnabled\n autoUpdateEnabled\n updateMapping {\n source\n sourcePath\n destinationColumn\n }\n jobs(pagination: { limit: 10 }) {\n lastEventAt\n status\n }\n automatedWebhooks\n webhookUrl\n ...DataSourceCard\n }\n }\n \n": types.AutoUpdateCreationReviewDocument,
"\n query ExternalDataSourceInspectPage($ID: ID!) {\n externalDataSource(pk: $ID) {\n id\n name\n dataType\n remoteUrl\n crmType\n connectionDetails {\n ... on AirtableSource {\n apiKey\n baseId\n tableId\n }\n ... on MailchimpSource {\n apiKey\n listId\n }\n ... on ActionNetworkSource {\n apiKey\n groupSlug\n }\n ... on TicketTailorSource {\n apiKey\n }\n }\n lastImportJob {\n id\n lastEventAt\n status\n }\n lastUpdateJob {\n id\n lastEventAt\n status\n }\n autoImportEnabled\n autoUpdateEnabled\n hasWebhooks\n allowUpdates\n automatedWebhooks\n webhookUrl\n webhookHealthcheck\n geographyColumn\n geographyColumnType\n geocodingConfig\n usesValidGeocodingConfig\n postcodeField\n firstNameField\n lastNameField\n fullNameField\n emailField\n phoneField\n addressField\n titleField\n descriptionField\n imageField\n startTimeField\n endTimeField\n publicUrlField\n socialUrlField\n canDisplayPointField\n isImportScheduled\n importProgress {\n id\n hasForecast\n status\n total\n succeeded\n estimatedFinishTime\n actualFinishTime\n inQueue\n numberOfJobsAheadInQueue\n sendEmail\n }\n isUpdateScheduled\n updateProgress {\n id\n hasForecast\n status\n total\n succeeded\n estimatedFinishTime\n actualFinishTime\n inQueue\n numberOfJobsAheadInQueue\n sendEmail\n }\n importedDataCount\n importedDataGeocodingRate\n regionCount: importedDataCountOfAreas(\n analyticalAreaType: european_electoral_region\n )\n constituencyCount: importedDataCountOfAreas(\n analyticalAreaType: parliamentary_constituency\n )\n ladCount: importedDataCountOfAreas(analyticalAreaType: admin_district)\n wardCount: importedDataCountOfAreas(analyticalAreaType: admin_ward)\n fieldDefinitions {\n label\n value\n description\n editable\n }\n updateMapping {\n source\n sourcePath\n destinationColumn\n }\n sharingPermissions {\n id\n }\n organisation {\n id\n name\n }\n }\n }\n": types.ExternalDataSourceInspectPageDocument,
"\n mutation DeleteUpdateConfig($id: String!) {\n deleteExternalDataSource(data: { id: $id }) {\n id\n }\n }\n": types.DeleteUpdateConfigDocument,
+ "\n mutation DeleteRecords($externalDataSourceId: String!) {\n deleteAllRecords(\n externalDataSourceId: $externalDataSourceId\n ) {\n id\n }\n }\n ": types.DeleteRecordsDocument,
"\n query ManageSourceSharing($externalDataSourceId: ID!) {\n externalDataSource(pk: $externalDataSourceId) {\n sharingPermissions {\n id\n organisationId\n organisation {\n name\n }\n externalDataSourceId\n visibilityRecordCoordinates\n visibilityRecordDetails\n deleted\n }\n }\n }\n ": types.ManageSourceSharingDocument,
"\n mutation UpdateSourceSharingObject(\n $data: SharingPermissionCUDInput!\n ) {\n updateSharingPermission(data: $data) {\n id\n organisationId\n externalDataSourceId\n visibilityRecordCoordinates\n visibilityRecordDetails\n deleted\n }\n }\n ": types.UpdateSourceSharingObjectDocument,
"\n mutation DeleteSourceSharingObject($pk: String!) {\n deleteSharingPermission(data: { id: $pk }) {\n id\n }\n }\n ": types.DeleteSourceSharingObjectDocument,
"\n mutation ImportData($id: String!) {\n importAll(externalDataSourceId: $id) {\n id\n externalDataSource {\n importedDataCount\n importProgress {\n status\n hasForecast\n id\n total\n succeeded\n failed\n estimatedFinishTime\n inQueue\n }\n }\n }\n }\n ": types.ImportDataDocument,
+ "\n mutation CancelImport($id: String!, $requestId: String!) {\n cancelImport(externalDataSourceId: $id, requestId: $requestId) {\n id\n }\n }\n ": types.CancelImportDocument,
"\n query ExternalDataSourceName($externalDataSourceId: ID!) {\n externalDataSource(pk: $externalDataSourceId) {\n name\n crmType\n dataType\n name\n remoteUrl\n }\n }\n": types.ExternalDataSourceNameDocument,
"\n mutation ShareDataSources(\n $fromOrgId: String!\n $permissions: [SharingPermissionInput!]!\n ) {\n updateSharingPermissions(\n fromOrgId: $fromOrgId\n permissions: $permissions\n ) {\n id\n sharingPermissions {\n id\n organisationId\n externalDataSourceId\n visibilityRecordCoordinates\n visibilityRecordDetails\n deleted\n }\n }\n }\n ": types.ShareDataSourcesDocument,
"\n query YourSourcesForSharing {\n myOrganisations {\n id\n name\n externalDataSources {\n id\n name\n crmType\n importedDataCount\n dataType\n fieldDefinitions {\n label\n editable\n }\n organisationId\n sharingPermissions {\n id\n organisationId\n externalDataSourceId\n visibilityRecordCoordinates\n visibilityRecordDetails\n deleted\n }\n }\n }\n }\n": types.YourSourcesForSharingDocument,
@@ -174,6 +176,10 @@ export function gql(source: "\n query ExternalDataSourceInspectPage($ID: ID!) {
* The gql function is used to parse GraphQL queries into a document that can be used by GraphQL clients.
*/
export function gql(source: "\n mutation DeleteUpdateConfig($id: String!) {\n deleteExternalDataSource(data: { id: $id }) {\n id\n }\n }\n"): (typeof documents)["\n mutation DeleteUpdateConfig($id: String!) {\n deleteExternalDataSource(data: { id: $id }) {\n id\n }\n }\n"];
+/**
+ * The gql function is used to parse GraphQL queries into a document that can be used by GraphQL clients.
+ */
+export function gql(source: "\n mutation DeleteRecords($externalDataSourceId: String!) {\n deleteAllRecords(\n externalDataSourceId: $externalDataSourceId\n ) {\n id\n }\n }\n "): (typeof documents)["\n mutation DeleteRecords($externalDataSourceId: String!) {\n deleteAllRecords(\n externalDataSourceId: $externalDataSourceId\n ) {\n id\n }\n }\n "];
/**
* The gql function is used to parse GraphQL queries into a document that can be used by GraphQL clients.
*/
@@ -190,6 +196,10 @@ export function gql(source: "\n mutation DeleteSourceSharingObject($pk:
* The gql function is used to parse GraphQL queries into a document that can be used by GraphQL clients.
*/
export function gql(source: "\n mutation ImportData($id: String!) {\n importAll(externalDataSourceId: $id) {\n id\n externalDataSource {\n importedDataCount\n importProgress {\n status\n hasForecast\n id\n total\n succeeded\n failed\n estimatedFinishTime\n inQueue\n }\n }\n }\n }\n "): (typeof documents)["\n mutation ImportData($id: String!) {\n importAll(externalDataSourceId: $id) {\n id\n externalDataSource {\n importedDataCount\n importProgress {\n status\n hasForecast\n id\n total\n succeeded\n failed\n estimatedFinishTime\n inQueue\n }\n }\n }\n }\n "];
+/**
+ * The gql function is used to parse GraphQL queries into a document that can be used by GraphQL clients.
+ */
+export function gql(source: "\n mutation CancelImport($id: String!, $requestId: String!) {\n cancelImport(externalDataSourceId: $id, requestId: $requestId) {\n id\n }\n }\n "): (typeof documents)["\n mutation CancelImport($id: String!, $requestId: String!) {\n cancelImport(externalDataSourceId: $id, requestId: $requestId) {\n id\n }\n }\n "];
/**
* The gql function is used to parse GraphQL queries into a document that can be used by GraphQL clients.
*/
diff --git a/nextjs/src/__generated__/graphql.ts b/nextjs/src/__generated__/graphql.ts
index d4e0dfdae..7c46eed9d 100644
--- a/nextjs/src/__generated__/graphql.ts
+++ b/nextjs/src/__generated__/graphql.ts
@@ -1904,12 +1904,14 @@ export type MultiPolygonGeometry = {
export type Mutation = {
__typename?: 'Mutation';
addMember: Scalars['Boolean']['output'];
+ cancelImport: ExternalDataSourceAction;
createApiToken: ApiToken;
createChildPage: HubPage;
createExternalDataSource: CreateExternalDataSourceOutput;
createMapReport: CreateMapReportPayload;
createOrganisation: Membership;
createSharingPermission: SharingPermission;
+ deleteAllRecords: ExternalDataSource;
deleteExternalDataSource: ExternalDataSource;
deleteMapReport: MapReport;
deletePage: Scalars['Boolean']['output'];
@@ -1994,6 +1996,12 @@ export type MutationAddMemberArgs = {
};
+export type MutationCancelImportArgs = {
+ externalDataSourceId: Scalars['String']['input'];
+ requestId: Scalars['String']['input'];
+};
+
+
export type MutationCreateApiTokenArgs = {
expiryDays?: Scalars['Int']['input'];
};
@@ -2025,6 +2033,11 @@ export type MutationCreateSharingPermissionArgs = {
};
+export type MutationDeleteAllRecordsArgs = {
+ externalDataSourceId: Scalars['String']['input'];
+};
+
+
export type MutationDeleteExternalDataSourceArgs = {
data: IdObject;
};
@@ -2364,6 +2377,7 @@ export type PostcodesIoResult = {
};
export enum ProcrastinateJobStatus {
+ Cancelled = 'cancelled',
Doing = 'doing',
Failed = 'failed',
Succeeded = 'succeeded',
@@ -3223,6 +3237,13 @@ export type DeleteUpdateConfigMutationVariables = Exact<{
export type DeleteUpdateConfigMutation = { __typename?: 'Mutation', deleteExternalDataSource: { __typename?: 'ExternalDataSource', id: any } };
+export type DeleteRecordsMutationVariables = Exact<{
+ externalDataSourceId: Scalars['String']['input'];
+}>;
+
+
+export type DeleteRecordsMutation = { __typename?: 'Mutation', deleteAllRecords: { __typename?: 'ExternalDataSource', id: any } };
+
export type ManageSourceSharingQueryVariables = Exact<{
externalDataSourceId: Scalars['ID']['input'];
}>;
@@ -3251,6 +3272,14 @@ export type ImportDataMutationVariables = Exact<{
export type ImportDataMutation = { __typename?: 'Mutation', importAll: { __typename?: 'ExternalDataSourceAction', id: string, externalDataSource: { __typename?: 'ExternalDataSource', importedDataCount: number, importProgress?: { __typename?: 'BatchJobProgress', status: ProcrastinateJobStatus, hasForecast: boolean, id: string, total?: number | null, succeeded?: number | null, failed?: number | null, estimatedFinishTime?: any | null, inQueue: boolean } | null } } };
+export type CancelImportMutationVariables = Exact<{
+ id: Scalars['String']['input'];
+ requestId: Scalars['String']['input'];
+}>;
+
+
+export type CancelImportMutation = { __typename?: 'Mutation', cancelImport: { __typename?: 'ExternalDataSourceAction', id: string } };
+
export type ExternalDataSourceNameQueryVariables = Exact<{
externalDataSourceId: Scalars['ID']['input'];
}>;
@@ -3639,10 +3668,12 @@ export const CreateSourceDocument = {"kind":"Document","definitions":[{"kind":"O
export const AutoUpdateCreationReviewDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"AutoUpdateCreationReview"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"ID"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"externalDataSource"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"pk"},"value":{"kind":"Variable","name":{"kind":"Name","value":"ID"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"geographyColumn"}},{"kind":"Field","name":{"kind":"Name","value":"geographyColumnType"}},{"kind":"Field","name":{"kind":"Name","value":"geocodingConfig"}},{"kind":"Field","name":{"kind":"Name","value":"usesValidGeocodingConfig"}},{"kind":"Field","name":{"kind":"Name","value":"dataType"}},{"kind":"Field","name":{"kind":"Name","value":"crmType"}},{"kind":"Field","name":{"kind":"Name","value":"autoImportEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"autoUpdateEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"updateMapping"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"source"}},{"kind":"Field","name":{"kind":"Name","value":"sourcePath"}},{"kind":"Field","name":{"kind":"Name","value":"destinationColumn"}}]}},{"kind":"Field","name":{"kind":"Name","value":"jobs"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"pagination"},"value":{"kind":"ObjectValue","fields":[{"kind":"ObjectField","name":{"kind":"Name","value":"limit"},"value":{"kind":"IntValue","value":"10"}}]}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"lastEventAt"}},{"kind":"Field","name":{"kind":"Name","value":"status"}}]}},{"kind":"Field","name":{"kind":"Name","value":"automatedWebhooks"}},{"kind":"Field","name":{"kind":"Name","value":"webhookUrl"}},{"kind":"FragmentSpread","name":{"kind":"Name","value":"DataSourceCard"}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"DataSourceCard"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ExternalDataSource"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"dataType"}},{"kind":"Field","name":{"kind":"Name","value":"crmType"}},{"kind":"Field","name":{"kind":"Name","value":"automatedWebhooks"}},{"kind":"Field","name":{"kind":"Name","value":"autoImportEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"autoUpdateEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"updateMapping"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"source"}},{"kind":"Field","name":{"kind":"Name","value":"sourcePath"}},{"kind":"Field","name":{"kind":"Name","value":"destinationColumn"}}]}},{"kind":"Field","name":{"kind":"Name","value":"jobs"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"pagination"},"value":{"kind":"ObjectValue","fields":[{"kind":"ObjectField","name":{"kind":"Name","value":"limit"},"value":{"kind":"IntValue","value":"10"}}]}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"lastEventAt"}},{"kind":"Field","name":{"kind":"Name","value":"status"}}]}},{"kind":"Field","name":{"kind":"Name","value":"sharingPermissions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"organisation"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}}]}}]}}]} as unknown as DocumentNode