From 702504a09b759f7c8b4224e16e9753c52b3bc00c Mon Sep 17 00:00:00 2001 From: Joaquim d'Souza Date: Thu, 19 Dec 2024 02:25:28 +0100 Subject: [PATCH] feat: dont create actionnetwork rows in test_sources as they cant be deleted --- hub/graphql/types/model_types.py | 10 +- hub/models.py | 3 +- hub/parsons/action_network/action_network.py | 6 +- hub/tests/test_sources.py | 301 +++++++++++++------ 4 files changed, 220 insertions(+), 100 deletions(-) diff --git a/hub/graphql/types/model_types.py b/hub/graphql/types/model_types.py index 12656fcbc..be09f0b03 100644 --- a/hub/graphql/types/model_types.py +++ b/hub/graphql/types/model_types.py @@ -593,21 +593,17 @@ async def gss_area(self, info: Info) -> Optional[Area]: @strawberry_django.type(models.GenericData, filters=CommonDataFilter) class GroupedData: label: Optional[str] - # Provide area_type if gss code is not unique (e.g. WMC and WMC23 constituencies) - area_type: Optional[str] = None + # Provide filter if gss code is not unique (e.g. WMC and WMC23 constituencies) + area_type_filter: Optional["AreaTypeFilter"] = None gss: Optional[str] area_data: Optional[strawberry.Private[Area]] = None imported_data: Optional[JSON] = None - area_type_filter: Optional["AreaTypeFilter"] = None @strawberry_django.field async def gss_area(self, info: Info) -> Optional[Area]: if self.area_data is not None: return self.area_data - if self.area_type is not None: - filters = {"area_type__code": self.area_type} - else: - filters = {} + filters = self.area_type_filter.query_filter if self.area_type_filter else {} loader = FieldDataLoaderFactory.get_loader_class( models.Area, field="gss", filters=filters ) diff --git a/hub/models.py b/hub/models.py index 5576ed5dd..1c7f91f6f 100644 --- a/hub/models.py +++ b/hub/models.py @@ -2830,7 +2830,8 @@ def get_record_id(self, record): return record["id"] def get_record_field(self, record, field, field_type=None): - d = record["fields"].get(str(field), None) + record_dict = record["fields"] if "fields" in record else record + d = record_dict.get(str(field), None) if field_type == "image_field" and d is not None and len(d) > 0: # TODO: implement image handling # e.g. [{'id': 'attDWjeMhUfNMTqRG', 'width': 2200, 'height': 1518, 'url': 'https://v5.airtableusercontent.com/v3/u/27/27/1712044800000/CxNHcR-sBRUhrWt_54_NFA/wcYpoqFV5W_wRmVwh2RM8qs-mJkwwHkQLZuhtf7rFk5-34gILMXJeIYg9vQMcTtgSEd1dDb7lU0CrgJldTcZBN9VyaTU0IkYiw1e5PzTs8ZsOEmA6wrva7UavQCnoacL8b7yUt4ZuWWhna8wzZD2MTZC1K1C1wLkfA1UyN76ZDO-Q6WkBjgg5uZv7rtXlhj9/WL6lQJQAHKXqA9J1YIteSJ3J0Yepj69c55PducG607k' diff --git a/hub/parsons/action_network/action_network.py b/hub/parsons/action_network/action_network.py index 8dba67633..baaa770ee 100644 --- a/hub/parsons/action_network/action_network.py +++ b/hub/parsons/action_network/action_network.py @@ -155,14 +155,14 @@ def _get_generator(self, object_name, limit=None, per_page=25, filter=None): while True: response = self._get_page(object_name, page, per_page, filter=filter) page = page + 1 - response_list = response["_embedded"][list(response["_embedded"])[0]] - if not response_list: - return + response_list = response["_embedded"][list(response["_embedded"])[0]] or [] for item in response_list: yield item count = count + 1 if limit and count >= limit: return + if len(response_list) < per_page: + return # Advocacy Campaigns def get_advocacy_campaigns(self, limit=None, per_page=25, page=None, filter=None): diff --git a/hub/tests/test_sources.py b/hub/tests/test_sources.py index d68c6e4b1..ca817cc76 100644 --- a/hub/tests/test_sources.py +++ b/hub/tests/test_sources.py @@ -61,12 +61,12 @@ def tearDown(self) -> None: self.source.teardown_unused_webhooks(force=True) return super().tearDown() - def create_test_record(self, record: models.ExternalDataSource.CUDRecord): + async def create_test_record(self, record: models.ExternalDataSource.CUDRecord): record = self.source.create_one(record) self.records_to_delete.append((self.source.get_record_id(record), self.source)) return record - def create_many_test_records( + async def create_many_test_records( self, records: List[models.ExternalDataSource.CUDRecord] ): records = self.source.create_many(records) @@ -195,7 +195,7 @@ async def test_import_many(self): self.assertEqual(len(df.index), import_count) async def test_fetch_one(self): - record = self.create_test_record( + record = await self.create_test_record( models.ExternalDataSource.CUDRecord( email=f"eh{randint(0, 1000)}sp@gmail.com", postcode="EH99 1SP", @@ -220,16 +220,37 @@ async def test_fetch_one(self): ) async def test_fetch_many(self): - now = str(datetime.now().timestamp()) test_record_data = [ models.ExternalDataSource.CUDRecord( - postcode=now + "11111", email=now + "11111@gmail.com", data={} + postcode="E5 0AA", + email=f"E{randint(0, 1000)}AA@gmail.com", + data=( + { + "addr1": "Millfields Rd", + "city": "London", + "state": "London", + "country": "GB", + } + if isinstance(self.source, models.MailchimpSource) + else {} + ), ), models.ExternalDataSource.CUDRecord( - postcode=now + "22222", email=now + "22222@gmail.com", data={} + postcode="E10 6EF", + email=f"E{randint(0, 1000)}EF@gmail.com", + data=( + { + "addr1": "123 Colchester Rd", + "city": "London", + "state": "London", + "country": "GB", + } + if isinstance(self.source, models.MailchimpSource) + else {} + ), ), ] - records = self.create_many_test_records(test_record_data) + records = await self.create_many_test_records(test_record_data) record_ids = [self.source.get_record_id(record) for record in records] assert len(record_ids) == 2 @@ -254,8 +275,10 @@ async def test_fetch_many(self): for test_record in test_record_data: record = next( filter( - lambda r: self.source.get_record_field(r, self.source.email_field) - == test_record["email"], + lambda r: self.source.get_record_field( + r, self.source.postcode_field + ) + == test_record["postcode"], records, ), None, @@ -263,7 +286,7 @@ async def test_fetch_many(self): self.assertIsNotNone(record) async def test_refresh_one(self): - record = self.create_test_record( + record = await self.create_test_record( models.ExternalDataSource.CUDRecord( email=f"eh{randint(0, 1000)}sp@gmail.com", postcode="EH99 1SP", @@ -315,7 +338,7 @@ async def test_pivot_table(self): [self.custom_data_layer.get_record_id(record) for record in records] ) # Add a test record - record = self.create_test_record( + record = await self.create_test_record( models.ExternalDataSource.CUDRecord( email=f"NE{randint(0, 1000)}DD@gmail.com", postcode="NE12 6DD", @@ -348,10 +371,10 @@ async def test_pivot_table(self): ) async def test_refresh_many(self): - records = self.create_many_test_records( + records = await self.create_many_test_records( [ models.ExternalDataSource.CUDRecord( - postcode="G11 5RD", + postcode="E10 6EF", email=f"gg{randint(0, 1000)}rardd@gmail.com", data=( { @@ -365,7 +388,7 @@ async def test_refresh_many(self): ), ), models.ExternalDataSource.CUDRecord( - postcode="G42 8PH", + postcode="E5 0AA", email=f"ag{randint(0, 1000)}rwefw@gmail.com", data=( { @@ -392,29 +415,67 @@ async def test_refresh_many(self): for record in records: if ( self.source.get_record_field(record, self.source.geography_column) - == "G11 5RD" + == "E5 0AA" ): self.assertEqual( self.source.get_record_field(record, self.constituency_field), - "Glasgow West", + "Hackney North and Stoke Newington", ) elif ( self.source.get_record_field(record, self.source.geography_column) - == "G42 8PH" + == "E10 6EF" ): self.assertEqual( self.source.get_record_field(record, self.constituency_field), - "Glasgow South", + "Leyton and Wanstead", ) else: self.fail() - async def test_analytics(self): + async def test_enrichment_electoral_commission(self): """ - This is testing the ability to get analytics from the data source + This is testing the ability to enrich data from the data source + using a third party source + """ + # Add a test record + record = await self.create_test_record( + models.ExternalDataSource.CUDRecord( + email=f"NE{randint(0, 1000)}DD@gmail.com", + postcode="DH1 1AE", + data=( + { + "addr1": "38 Swinside Dr", + "city": "Durham", + "state": "Durham", + "country": "GB", + } + if isinstance(self.source, models.MailchimpSource) + else {} + ), + ) + ) + mapped_member = await self.source.map_one( + record, + loaders=await self.source.get_loaders(), + mapping=[ + models.UpdateMapping( + source="electoral_commission_postcode_lookup", + source_path="electoral_services.name", + destination_column="electoral service", + ) + ], + ) + self.assertEqual( + mapped_member["update_fields"]["electoral service"], + "Durham County Council", + ) + + async def test_analytics_counts(self): + """ + This is testing the ability to get record counts from the data source """ # Add some test data - self.create_many_test_records( + created_records = await self.create_many_test_records( [ models.ExternalDataSource.CUDRecord( postcode="E5 0AA", @@ -447,13 +508,14 @@ async def test_analytics(self): ] ) # import - records = await self.source.fetch_all() + records = await self.source.fetch_many( + [self.source.get_record_id(record) for record in created_records] + ) await self.source.import_many( [self.source.get_record_id(record) for record in records] ) # check analytics analytics = self.source.imported_data_count_by_constituency() - # convert query set to list (is there a better way?) analytics = await sync_to_async(list)(analytics) self.assertGreaterEqual(len(analytics), 2) constituencies_in_report = [a["label"] for a in analytics] @@ -466,6 +528,110 @@ async def test_analytics(self): elif a["label"] == "Leyton and Wanstead": self.assertGreaterEqual(a["count"], 1) + analytics = self.source.imported_data_count_by_area("admin_district") + analytics = await sync_to_async(list)(analytics) + self.assertGreaterEqual(len(analytics), 2) + constituencies_in_report = [a["label"] for a in analytics] + + self.assertIn("Hackney", constituencies_in_report) + self.assertIn("Waltham Forest", constituencies_in_report) + for a in analytics: + if a["label"] == "Hackney": + self.assertGreaterEqual(a["count"], 1) + elif a["label"] == "Waltham Forest": + self.assertGreaterEqual(a["count"], 1) + + async def test_analytics_imported_data(self): + """ + This is testing the ability to get record data from the data source + """ + # Add some test data + created_records = await self.create_many_test_records( + [ + models.ExternalDataSource.CUDRecord( + postcode="E5 0AA", + email=f"E{randint(0, 1000)}AA@gmail.com", + data=( + { + "addr1": "Millfields Rd", + "city": "London", + "state": "London", + "country": "GB", + } + if isinstance(self.source, models.MailchimpSource) + else {} + ), + ), + models.ExternalDataSource.CUDRecord( + postcode="E5 0AB", + email=f"E{randint(0, 1000)}AA@gmail.com", + data=( + { + "addr1": "Millfields Rd", + "city": "London", + "state": "London", + "country": "GB", + } + if isinstance(self.source, models.MailchimpSource) + else {} + ), + ), + models.ExternalDataSource.CUDRecord( + postcode="E10 6EF", + email=f"E{randint(0, 1000)}EF@gmail.com", + data=( + { + "addr1": "123 Colchester Rd", + "city": "London", + "state": "London", + "country": "GB", + } + if isinstance(self.source, models.MailchimpSource) + else {} + ), + ), + ] + ) + # import + records = await self.source.fetch_many( + [self.source.get_record_id(record) for record in created_records] + ) + await self.source.import_many( + [self.source.get_record_id(record) for record in records] + ) + # check analytics + analytics = self.source.imported_data_by_area("parliamentary_constituency") + analytics = await sync_to_async(list)(analytics) + self.assertGreaterEqual(len(analytics), 3) + constituencies_in_report = [a["label"] for a in analytics] + + self.assertIn("Hackney North and Stoke Newington", constituencies_in_report) + self.assertIn("Leyton and Wanstead", constituencies_in_report) + for a in analytics: + postcode = self.source.get_record_field( + a["imported_data"], self.source.postcode_field + ) + if a["label"] == "Hackney North and Stoke Newington": + self.assertIn(postcode, ["E5 0AA", "E5 0AB"]) + elif a["label"] == "Leyton and Wanstead": + self.assertEqual(postcode, "E10 6EF") + + analytics = self.source.imported_data_by_area("admin_district") + analytics = await sync_to_async(list)(analytics) + self.assertGreaterEqual(len(analytics), 3) + constituencies_in_report = [a["label"] for a in analytics] + + self.assertIn("Hackney", constituencies_in_report) + self.assertIn("Waltham Forest", constituencies_in_report) + for a in analytics: + postcode = self.source.get_record_field( + a["imported_data"], self.source.postcode_field + ) + if a["label"] == "Hackney": + self.assertIn(postcode, ["E5 0AA", "E5 0AB"]) + elif a["label"] == "Waltham Forest": + self.assertEqual(postcode, "E10 6EF") + class TestAirtableSource(TestExternalDataSource, TestCase): def create_test_source(self, name="My test Airtable member list"): @@ -496,35 +662,6 @@ def create_test_source(self, name="My test Airtable member list"): ) return self.source - async def test_enrichment_electoral_commission(self): - """ - This is testing the ability to enrich data from the data source - using a third party source - """ - # Add a test record - record = self.create_test_record( - models.ExternalDataSource.CUDRecord( - email=f"NE{randint(0, 1000)}DD@gmail.com", - postcode="DH1 1AE", - data={}, - ) - ) - mapped_member = await self.source.map_one( - record, - loaders=await self.source.get_loaders(), - mapping=[ - models.UpdateMapping( - source="electoral_commission_postcode_lookup", - source_path="electoral_services.name", - destination_column="electoral service", - ) - ], - ) - self.assertEqual( - mapped_member["update_fields"]["electoral service"], - "Durham County Council", - ) - class TestMailchimpSource(TestExternalDataSource, TestCase): constituency_field = "CONSTITUEN" @@ -587,44 +724,32 @@ def create_test_source(self, name="My test AN member list"): ) return self.source + async def create_test_record(self, record: models.ExternalDataSource.CUDRecord): + records = await self.create_many_test_records([record]) + return records[0] + + async def create_many_test_records( + self, records: List[models.ExternalDataSource.CUDRecord] + ): + # don't create records, and return existing records + # this is because Action Network records can't be deleted + postcodes_to_ids = { + "EH99 1SP": "c6d37304-200c-44b4-8eda-04a03e706531", + "NE12 6DD": "2574d845-f5bb-4ba2-af9b-a712d10119b1", + "DH1 1AE": "42fe3b4a-f445-47ce-ba81-7ec38d95dc70", + "E10 6EF": "d88da43f-8984-41d8-80fa-4f9fbb3d6006", + "E5 0AA": "ad6228a2-74c1-48fd-85ee-90eafbaca397", + "E5 0AB": "b762c93b-a23d-45c8-85c4-0d20c3c8a9e5", + } + records = await self.source.fetch_many( + [postcodes_to_ids[record["postcode"]] for record in records] + ) + return records + async def test_fetch_page(self): """ Ensure that fetching page-by-page gives the same count as fetching all. """ - # Add some test data - self.create_many_test_records( - [ - models.ExternalDataSource.CUDRecord( - postcode="E5 0AA", - email=f"E{randint(0, 1000)}AA@gmail.com", - data=( - { - "addr1": "Millfields Rd", - "city": "London", - "state": "London", - "country": "GB", - } - if isinstance(self.source, models.MailchimpSource) - else {} - ), - ), - models.ExternalDataSource.CUDRecord( - postcode="E10 6EF", - email=f"E{randint(0, 1000)}EF@gmail.com", - data=( - { - "addr1": "123 Colchester Rd", - "city": "London", - "state": "London", - "country": "GB", - } - if isinstance(self.source, models.MailchimpSource) - else {} - ), - ), - ] - ) - all_records = await self.source.fetch_all() all_records = list(all_records) paged_records = [] @@ -683,7 +808,7 @@ async def test_fetch_all(self): postcode=now + "22222", email=now + "22222@gmail.com", data={} ), ] - self.create_many_test_records(test_record_data) + await self.create_many_test_records(test_record_data) # Test this functionality records = await self.source.fetch_all() @@ -692,8 +817,6 @@ async def test_fetch_all(self): # Assumes there were 4 records in the test data source before this test ran assert len(records) == 6 - # Check the email field instead of postcode, because Mailchimp doesn't set - # the postcode without a full address, which is not present in this test for test_record in test_record_data: record = next( filter(