From b50c38cab10dca5cf436055cbaafde9d302e85eb Mon Sep 17 00:00:00 2001 From: prijendev Date: Tue, 6 Sep 2022 14:55:17 +0530 Subject: [PATCH 1/5] Fixed bookmark issue for conversations stream. --- tap_freshdesk/streams.py | 11 +++ tap_freshdesk/test_conversations.py | 52 ++++++++++ tap_freshdesk/test_streams.py | 147 ++++++++++++++++++++++++++++ 3 files changed, 210 insertions(+) create mode 100644 tap_freshdesk/test_conversations.py create mode 100644 tap_freshdesk/test_streams.py diff --git a/tap_freshdesk/streams.py b/tap_freshdesk/streams.py index d8db0c7..19ddbe7 100644 --- a/tap_freshdesk/streams.py +++ b/tap_freshdesk/streams.py @@ -88,6 +88,9 @@ def build_url(self, base_url, *args): """ return base_url + '/api/v2/'+ self.path.format(*args) + def add_fields_at_1st_level(self, record): + pass + def write_records(self, catalog, state, selected_streams, start_date, data, max_bookmark, client, streams_to_sync, child_max_bookmarks, predefined_filter=None): """ Transform the chunk of records according to the schema and write the records based on the bookmark. @@ -109,6 +112,7 @@ def write_records(self, catalog, state, selected_streams, start_date, data, max_ extraction_time = singer.utils.now() stream_metadata = singer.metadata.to_map(stream_catalog['metadata']) for row in data: + self.add_fields_at_1st_level(row) if self.tap_stream_id in selected_streams and row[self.replication_keys[0]] >= bookmark: # Custom fields are expected to be strings, but sometimes the API sends # booleans. We cast those to strings to match the schema. @@ -285,6 +289,13 @@ class Conversations(ChildStream): path = 'tickets/{}/conversations' parent = 'tickets' + def add_fields_at_1st_level(self, record): + """ + Overwrite updated__at value. + """ + if record.get("last_edited_at"): + record["updated_at"] = max(record["updated_at"], record["last_edited_at"]) + class SatisfactionRatings(ChildStream): """ diff --git a/tap_freshdesk/test_conversations.py b/tap_freshdesk/test_conversations.py new file mode 100644 index 0000000..947e8c4 --- /dev/null +++ b/tap_freshdesk/test_conversations.py @@ -0,0 +1,52 @@ +import unittest +from unittest import mock +from parameterized import parameterized +from tap_freshdesk.streams import Tickets + + +class TestConversationSync(unittest.TestCase): + """ + Test `sync_obj` method of conversation stream. + If `last_edited_at` field is not null then `updated_at` will be overwritten by `last_edited_at`. + """ + start_date = "2019-06-01T00:00:00Z" + + responses_1 = [ + [{"id": "33", "updated_at": "2020-03-01T00:00:00Z"}], # Tickets Response + [{"id": "44", "updated_at": "2020-04-01T00:00:00Z", "last_edited_at": "2020-05-01T00:00:00Z"}], # conversations Response + [], [] + ] + + responses_2 = [ + [{"id": "33", "updated_at": "2020-03-01T00:00:00Z"}], # Tickets Response + [{"id": "44", "updated_at": "2020-04-01T00:00:00Z", "last_edited_at": None}], # conversations Response + [], [] + ] + + @parameterized.expand([ + ["with last_edited_at value", responses_1, "2020-05-01T00:00:00Z"], + ["with null last_edited_at", responses_2, "2020-04-01T00:00:00Z"], + ]) + @mock.patch("singer.write_record") + @mock.patch("singer.write_bookmark") + def test_stream(self, name, responses, expected_updated_at, mock_write_bookmark, mock_write_record): + """ + Test that the stream is writing the expected record. + """ + stream = Tickets() + state = {} + client = mock.Mock() + client.base_url = "" + client.request.side_effect = responses + + # Record with expected `updated_at` value + expected_record = {**responses[1][0], "updated_at": expected_updated_at} + catalog = [ + {"schema":{}, "tap_stream_id": "tickets", "metadata": []}, + {"schema":{}, "tap_stream_id": "conversations", "metadata": []} + ] + + stream.sync_obj(state, self.start_date, client, catalog, ["conversations"], ["tickets"]) + + # Verify that the expected record is written + mock_write_record.assert_called_with(mock.ANY, expected_record, time_extracted=mock.ANY) diff --git a/tap_freshdesk/test_streams.py b/tap_freshdesk/test_streams.py new file mode 100644 index 0000000..cb399b8 --- /dev/null +++ b/tap_freshdesk/test_streams.py @@ -0,0 +1,147 @@ +import unittest +from unittest import mock +from parameterized import parameterized +from tap_freshdesk.streams import Agents, Tickets + + +class TestSyncObj(unittest.TestCase): + """ + Test `sync_obj` method of stream. + """ + + start_date = "2019-06-01T00:00:00Z" + only_parent_response = [ + [{"id": i, "updated_at": f"2020-0{i}-01T00:00:00Z"} for i in [1,5,2]], # Tickets Response + [{"id": "33", "updated_at": f"2020-03-01T00:00:00Z"}], # Deleted tickets Response + [{"id": "55", "updated_at": f"2020-04-01T00:00:00Z"}], # Spam tickets Response + ] + written_states_1 = { + "tickets": "2020-05-01T00:00:00Z", + "tickets_deleted": "2020-03-01T00:00:00Z", + "tickets_spam": "2020-04-01T00:00:00Z", + } + with_child_response = [ + [{"id": i, "updated_at": f"2020-0{i}-01T00:00:00Z"} for i in [1,5,2]], # Tickets Response + [{"id": i, "updated_at": f"2020-0{i}-01T00:00:00Z"} for i in [2,4]], # conversations Response + [{"id": "33", "updated_at": "2020-03-01T00:00:00Z"}], # conversations Response + [{"id": "55", "updated_at": "2020-04-01T00:00:00Z"}], # conversations Response + [],[] # Deleted/Spam tickets response + ] + written_states_2 = { + "conversations": "2020-04-01T00:00:00Z", + } + written_states_3 = { + "tickets": "2020-05-01T00:00:00Z", + "conversations": "2020-04-01T00:00:00Z", + } + expected_state_1 = { + "conversations": {"updated_at": "2020-04-01T00:00:00Z"}, + "tickets": {"updated_at": "2020-03-15T00:00:00Z"}, + "tickets_deleted": {"updated_at": "2020-05-01T00:00:00Z"}, + "tickets_spam": {"updated_at": "2020-04-01T00:00:00Z"} + } + expected_state_2 = {'conversations': {'updated_at': '2020-04-01T00:00:00Z'}, + 'tickets': {'updated_at': '2019-06-01T00:00:00Z'}, + 'tickets_deleted': {'updated_at': '2020-05-01T00:00:00Z'}, + 'tickets_spam': {'updated_at': '2020-04-01T00:00:00Z'}} + expected_state_3 = { + **expected_state_1, + "tickets": {"updated_at": "2020-03-16T00:00:00Z"}, + } + + @parameterized.expand([ + ["parent_selected", ["tickets"], ["tickets"], only_parent_response, 5, written_states_1], + ["child_selected", ["conversations"], ["tickets", "conversations"], with_child_response, 4, written_states_2], + ["parent_child_both_selected", ["tickets", "conversations"], ["tickets", "conversations"], with_child_response, 7, written_states_3], + ]) + @mock.patch("singer.write_record") + @mock.patch("singer.write_bookmark") + def test_stream(self, name, selected_streams, streams_to_sync, responses, written_records, written_states, mock_write_bookmark, mock_write_record): + """ + Test that stream is writing records and bookmarks only if selected. + """ + stream = Tickets() + state = {} + client = mock.Mock() + client.base_url = "" + client.request.side_effect = responses + catalog = [ + {"schema":{}, "tap_stream_id": "tickets", "metadata": []}, + {"schema":{}, "tap_stream_id": "conversations", "metadata": []} + ] + + stream.sync_obj(state, self.start_date, client, catalog, selected_streams, streams_to_sync) + + # Verify expected records are written + self.assertEqual(mock_write_record.call_count, written_records) + + # Verify max bookmark is updated for all selected streams + for stream, bookmark in written_states.items(): + mock_write_bookmark.assert_any_call({}, stream, "updated_at", bookmark) + + + @parameterized.expand([ + ["without_state", dict(), expected_state_1, 13], + ["with_parent_state", {"bookmarks": {"tickets": {"updated_at": "2020-03-16T00:00:00Z"}}}, expected_state_2, 10], + ["with_child_state", {"bookmarks": {"conversations": {"updated_at": "2020-03-23T00:00:00Z"}}}, expected_state_1, 8], + ["with_both_state", {"bookmarks": {"tickets": {"updated_at": "2020-03-16T00:00:00Z"}, "conversations": {"updated_at": "2020-03-23T00:00:00Z"}}}, expected_state_3, 5], + ]) + @mock.patch("singer.write_record") + def test_parent_child_both_selected(self, name, state, expected_state, written_records, mock_write_record): + """ + Test parent and child streams both selected in given conditions: + - Without state + - With only parent bookmark + - With only child bookmark + - With both parent and child bookmark + """ + stream = Tickets() + client = mock.Mock() + client.base_url = "" + client.request.side_effect = [ + [{"id": i, "updated_at": f"2020-03-{i}T00:00:00Z"} for i in [11,15,12]], # Tickets Response + [{"id": 10+i, "updated_at": f"2020-03-{i}T00:00:00Z"} for i in [13,24]], # conversations Response + [{"id": 13, "updated_at": "2020-03-01T00:00:00Z", "last_edited_at": "2020-03-02T00:00:00Z"}], # conversations Response + [{"id": 95, "updated_at": "2020-04-01T00:00:00Z"}], # conversations Response + [{"id": 73, "updated_at": "2020-05-01T00:00:00Z"}], # Deleted tickets response + [{"id": 30+i, "updated_at": f"2020-03-{i}T00:00:00Z"}for i in [22,10]], # conversations response + [{"id": 43, "updated_at": "2020-04-01T00:00:00Z"}], # Spam tickets response + [{"id": 50+i, "updated_at": f"2020-03-{i}T00:00:00Z"}for i in [12,25]], # conversations response + ] + catalog = [ + {"schema":{}, "tap_stream_id": "tickets", "metadata": []}, + {"schema":{}, "tap_stream_id": "conversations", "metadata": []} + ] + + stream.sync_obj(state, self.start_date, client, catalog, ["tickets", "conversations"], ["tickets", "conversations"]) + self.assertEqual(mock_write_record.call_count, written_records) + self.assertDictEqual(state, {"bookmarks": expected_state}) + + +class TestSyncTransformDict(unittest.TestCase): + """ + Test `transform_dict` method of stream class. + """ + + stream = Agents() + expected_list_1 = [{"name": "Agency", "value": "Justice League"}, + {"name": "Department", "value": "Superhero"}] + expected_list_2 = [{"key": "Agency", "data": "Justice League"}, + {"key": "Department", "data": "Superhero"}] + expected_list_3 = [{"name": "Agency", "value": "justice league"}, + {"name": "Department", "value": "superhero"}] + @parameterized.expand([ + ["coverting_dict_to_list", {"Agency": "Justice League", "Department": "Superhero"}, expected_list_1, {}], + ["With_custom_keys", {"Agency": "Justice League", "Department": "Superhero"}, expected_list_2, {"key_key":"key", "value_key":"data"}], + ["With_string_value", {"Agency": "Justice League", "Department": "Superhero"}, expected_list_3, {"force_str": True}], + ]) + def test_transform(self, name, dictionary, expected_list, kwargs): + """ + Test that the dictionary is transformed as per given conditions: + - Value is a lowercase string when force_str: True + - Key-Values can be customized by passing in args + """ + returned_list = self.stream.transform_dict(dictionary, **kwargs) + + # Verify returned list is expected + self.assertEqual(returned_list, expected_list) From 4d17e3183bef12fb36105034f938be406328422e Mon Sep 17 00:00:00 2001 From: prijendev Date: Tue, 6 Sep 2022 15:04:46 +0530 Subject: [PATCH 2/5] Fixed bookmark issue for conversations stream. --- tap_freshdesk/streams.py | 11 ++++++ tests/unittests/test_conversations.py | 52 +++++++++++++++++++++++++++ tests/unittests/test_streams.py | 4 +-- 3 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 tests/unittests/test_conversations.py diff --git a/tap_freshdesk/streams.py b/tap_freshdesk/streams.py index d8db0c7..19ddbe7 100644 --- a/tap_freshdesk/streams.py +++ b/tap_freshdesk/streams.py @@ -88,6 +88,9 @@ def build_url(self, base_url, *args): """ return base_url + '/api/v2/'+ self.path.format(*args) + def add_fields_at_1st_level(self, record): + pass + def write_records(self, catalog, state, selected_streams, start_date, data, max_bookmark, client, streams_to_sync, child_max_bookmarks, predefined_filter=None): """ Transform the chunk of records according to the schema and write the records based on the bookmark. @@ -109,6 +112,7 @@ def write_records(self, catalog, state, selected_streams, start_date, data, max_ extraction_time = singer.utils.now() stream_metadata = singer.metadata.to_map(stream_catalog['metadata']) for row in data: + self.add_fields_at_1st_level(row) if self.tap_stream_id in selected_streams and row[self.replication_keys[0]] >= bookmark: # Custom fields are expected to be strings, but sometimes the API sends # booleans. We cast those to strings to match the schema. @@ -285,6 +289,13 @@ class Conversations(ChildStream): path = 'tickets/{}/conversations' parent = 'tickets' + def add_fields_at_1st_level(self, record): + """ + Overwrite updated__at value. + """ + if record.get("last_edited_at"): + record["updated_at"] = max(record["updated_at"], record["last_edited_at"]) + class SatisfactionRatings(ChildStream): """ diff --git a/tests/unittests/test_conversations.py b/tests/unittests/test_conversations.py new file mode 100644 index 0000000..947e8c4 --- /dev/null +++ b/tests/unittests/test_conversations.py @@ -0,0 +1,52 @@ +import unittest +from unittest import mock +from parameterized import parameterized +from tap_freshdesk.streams import Tickets + + +class TestConversationSync(unittest.TestCase): + """ + Test `sync_obj` method of conversation stream. + If `last_edited_at` field is not null then `updated_at` will be overwritten by `last_edited_at`. + """ + start_date = "2019-06-01T00:00:00Z" + + responses_1 = [ + [{"id": "33", "updated_at": "2020-03-01T00:00:00Z"}], # Tickets Response + [{"id": "44", "updated_at": "2020-04-01T00:00:00Z", "last_edited_at": "2020-05-01T00:00:00Z"}], # conversations Response + [], [] + ] + + responses_2 = [ + [{"id": "33", "updated_at": "2020-03-01T00:00:00Z"}], # Tickets Response + [{"id": "44", "updated_at": "2020-04-01T00:00:00Z", "last_edited_at": None}], # conversations Response + [], [] + ] + + @parameterized.expand([ + ["with last_edited_at value", responses_1, "2020-05-01T00:00:00Z"], + ["with null last_edited_at", responses_2, "2020-04-01T00:00:00Z"], + ]) + @mock.patch("singer.write_record") + @mock.patch("singer.write_bookmark") + def test_stream(self, name, responses, expected_updated_at, mock_write_bookmark, mock_write_record): + """ + Test that the stream is writing the expected record. + """ + stream = Tickets() + state = {} + client = mock.Mock() + client.base_url = "" + client.request.side_effect = responses + + # Record with expected `updated_at` value + expected_record = {**responses[1][0], "updated_at": expected_updated_at} + catalog = [ + {"schema":{}, "tap_stream_id": "tickets", "metadata": []}, + {"schema":{}, "tap_stream_id": "conversations", "metadata": []} + ] + + stream.sync_obj(state, self.start_date, client, catalog, ["conversations"], ["tickets"]) + + # Verify that the expected record is written + mock_write_record.assert_called_with(mock.ANY, expected_record, time_extracted=mock.ANY) diff --git a/tests/unittests/test_streams.py b/tests/unittests/test_streams.py index 789bd6c..cb399b8 100644 --- a/tests/unittests/test_streams.py +++ b/tests/unittests/test_streams.py @@ -6,7 +6,7 @@ class TestSyncObj(unittest.TestCase): """ - Test `sync_obj` mehtod of stream. + Test `sync_obj` method of stream. """ start_date = "2019-06-01T00:00:00Z" @@ -101,7 +101,7 @@ def test_parent_child_both_selected(self, name, state, expected_state, written_r client.request.side_effect = [ [{"id": i, "updated_at": f"2020-03-{i}T00:00:00Z"} for i in [11,15,12]], # Tickets Response [{"id": 10+i, "updated_at": f"2020-03-{i}T00:00:00Z"} for i in [13,24]], # conversations Response - [{"id": 13, "updated_at": "2020-03-01T00:00:00Z"}], # conversations Response + [{"id": 13, "updated_at": "2020-03-01T00:00:00Z", "last_edited_at": "2020-03-02T00:00:00Z"}], # conversations Response [{"id": 95, "updated_at": "2020-04-01T00:00:00Z"}], # conversations Response [{"id": 73, "updated_at": "2020-05-01T00:00:00Z"}], # Deleted tickets response [{"id": 30+i, "updated_at": f"2020-03-{i}T00:00:00Z"}for i in [22,10]], # conversations response From 928e9987bc25889162e7f80f8bbfdf089ebf2608 Mon Sep 17 00:00:00 2001 From: prijendev Date: Wed, 21 Sep 2022 10:51:59 +0530 Subject: [PATCH 3/5] Added code comments. --- tap_freshdesk/streams.py | 2 ++ tap_freshdesk/test_conversations.py | 17 ++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/tap_freshdesk/streams.py b/tap_freshdesk/streams.py index 4551b9b..d5d5a52 100644 --- a/tap_freshdesk/streams.py +++ b/tap_freshdesk/streams.py @@ -320,6 +320,8 @@ def add_fields_at_1st_level(self, record): """ Overwrite updated__at value. """ + # For edited conversations `last_edited_at` value gets update instead of `updated_at` + # Hence `updated_at` will be overwritten if `last_edited_at` > `updated_at` if record.get("last_edited_at"): record["updated_at"] = max(record["updated_at"], record["last_edited_at"]) diff --git a/tap_freshdesk/test_conversations.py b/tap_freshdesk/test_conversations.py index 947e8c4..0070ca2 100644 --- a/tap_freshdesk/test_conversations.py +++ b/tap_freshdesk/test_conversations.py @@ -12,24 +12,27 @@ class TestConversationSync(unittest.TestCase): start_date = "2019-06-01T00:00:00Z" responses_1 = [ - [{"id": "33", "updated_at": "2020-03-01T00:00:00Z"}], # Tickets Response - [{"id": "44", "updated_at": "2020-04-01T00:00:00Z", "last_edited_at": "2020-05-01T00:00:00Z"}], # conversations Response + [{"id": "33", "updated_at": "2020-03-01T00:00:00Z"}], # Tickets Response + [{"id": "44", "updated_at": "2020-04-01T00:00:00Z", + "last_edited_at": "2020-05-01T00:00:00Z"}], # conversations Response [], [] ] responses_2 = [ - [{"id": "33", "updated_at": "2020-03-01T00:00:00Z"}], # Tickets Response - [{"id": "44", "updated_at": "2020-04-01T00:00:00Z", "last_edited_at": None}], # conversations Response + [{"id": "33", "updated_at": "2020-03-01T00:00:00Z"}], # Tickets Response + [{"id": "44", "updated_at": "2020-04-01T00:00:00Z", + "last_edited_at": None}], # conversations Response [], [] ] @parameterized.expand([ + # ["test_name", "responses", "expected_updated_at"] ["with last_edited_at value", responses_1, "2020-05-01T00:00:00Z"], ["with null last_edited_at", responses_2, "2020-04-01T00:00:00Z"], ]) @mock.patch("singer.write_record") @mock.patch("singer.write_bookmark") - def test_stream(self, name, responses, expected_updated_at, mock_write_bookmark, mock_write_record): + def test_stream(self, test_name, responses, expected_updated_at, mock_write_bookmark, mock_write_record): """ Test that the stream is writing the expected record. """ @@ -42,8 +45,8 @@ def test_stream(self, name, responses, expected_updated_at, mock_write_bookmark, # Record with expected `updated_at` value expected_record = {**responses[1][0], "updated_at": expected_updated_at} catalog = [ - {"schema":{}, "tap_stream_id": "tickets", "metadata": []}, - {"schema":{}, "tap_stream_id": "conversations", "metadata": []} + {"schema": {}, "tap_stream_id": "tickets", "metadata": []}, + {"schema": {}, "tap_stream_id": "conversations", "metadata": []} ] stream.sync_obj(state, self.start_date, client, catalog, ["conversations"], ["tickets"]) From d0cf18fc54b8b3d1c41077d11e483053b288900b Mon Sep 17 00:00:00 2001 From: prijendev Date: Wed, 21 Sep 2022 11:42:38 +0530 Subject: [PATCH 4/5] Removed unnecessary files. --- tap_freshdesk/test_conversations.py | 55 ---------- tap_freshdesk/test_streams.py | 147 -------------------------- tests/unittests/test_conversations.py | 17 +-- 3 files changed, 10 insertions(+), 209 deletions(-) delete mode 100644 tap_freshdesk/test_conversations.py delete mode 100644 tap_freshdesk/test_streams.py diff --git a/tap_freshdesk/test_conversations.py b/tap_freshdesk/test_conversations.py deleted file mode 100644 index 0070ca2..0000000 --- a/tap_freshdesk/test_conversations.py +++ /dev/null @@ -1,55 +0,0 @@ -import unittest -from unittest import mock -from parameterized import parameterized -from tap_freshdesk.streams import Tickets - - -class TestConversationSync(unittest.TestCase): - """ - Test `sync_obj` method of conversation stream. - If `last_edited_at` field is not null then `updated_at` will be overwritten by `last_edited_at`. - """ - start_date = "2019-06-01T00:00:00Z" - - responses_1 = [ - [{"id": "33", "updated_at": "2020-03-01T00:00:00Z"}], # Tickets Response - [{"id": "44", "updated_at": "2020-04-01T00:00:00Z", - "last_edited_at": "2020-05-01T00:00:00Z"}], # conversations Response - [], [] - ] - - responses_2 = [ - [{"id": "33", "updated_at": "2020-03-01T00:00:00Z"}], # Tickets Response - [{"id": "44", "updated_at": "2020-04-01T00:00:00Z", - "last_edited_at": None}], # conversations Response - [], [] - ] - - @parameterized.expand([ - # ["test_name", "responses", "expected_updated_at"] - ["with last_edited_at value", responses_1, "2020-05-01T00:00:00Z"], - ["with null last_edited_at", responses_2, "2020-04-01T00:00:00Z"], - ]) - @mock.patch("singer.write_record") - @mock.patch("singer.write_bookmark") - def test_stream(self, test_name, responses, expected_updated_at, mock_write_bookmark, mock_write_record): - """ - Test that the stream is writing the expected record. - """ - stream = Tickets() - state = {} - client = mock.Mock() - client.base_url = "" - client.request.side_effect = responses - - # Record with expected `updated_at` value - expected_record = {**responses[1][0], "updated_at": expected_updated_at} - catalog = [ - {"schema": {}, "tap_stream_id": "tickets", "metadata": []}, - {"schema": {}, "tap_stream_id": "conversations", "metadata": []} - ] - - stream.sync_obj(state, self.start_date, client, catalog, ["conversations"], ["tickets"]) - - # Verify that the expected record is written - mock_write_record.assert_called_with(mock.ANY, expected_record, time_extracted=mock.ANY) diff --git a/tap_freshdesk/test_streams.py b/tap_freshdesk/test_streams.py deleted file mode 100644 index cb399b8..0000000 --- a/tap_freshdesk/test_streams.py +++ /dev/null @@ -1,147 +0,0 @@ -import unittest -from unittest import mock -from parameterized import parameterized -from tap_freshdesk.streams import Agents, Tickets - - -class TestSyncObj(unittest.TestCase): - """ - Test `sync_obj` method of stream. - """ - - start_date = "2019-06-01T00:00:00Z" - only_parent_response = [ - [{"id": i, "updated_at": f"2020-0{i}-01T00:00:00Z"} for i in [1,5,2]], # Tickets Response - [{"id": "33", "updated_at": f"2020-03-01T00:00:00Z"}], # Deleted tickets Response - [{"id": "55", "updated_at": f"2020-04-01T00:00:00Z"}], # Spam tickets Response - ] - written_states_1 = { - "tickets": "2020-05-01T00:00:00Z", - "tickets_deleted": "2020-03-01T00:00:00Z", - "tickets_spam": "2020-04-01T00:00:00Z", - } - with_child_response = [ - [{"id": i, "updated_at": f"2020-0{i}-01T00:00:00Z"} for i in [1,5,2]], # Tickets Response - [{"id": i, "updated_at": f"2020-0{i}-01T00:00:00Z"} for i in [2,4]], # conversations Response - [{"id": "33", "updated_at": "2020-03-01T00:00:00Z"}], # conversations Response - [{"id": "55", "updated_at": "2020-04-01T00:00:00Z"}], # conversations Response - [],[] # Deleted/Spam tickets response - ] - written_states_2 = { - "conversations": "2020-04-01T00:00:00Z", - } - written_states_3 = { - "tickets": "2020-05-01T00:00:00Z", - "conversations": "2020-04-01T00:00:00Z", - } - expected_state_1 = { - "conversations": {"updated_at": "2020-04-01T00:00:00Z"}, - "tickets": {"updated_at": "2020-03-15T00:00:00Z"}, - "tickets_deleted": {"updated_at": "2020-05-01T00:00:00Z"}, - "tickets_spam": {"updated_at": "2020-04-01T00:00:00Z"} - } - expected_state_2 = {'conversations': {'updated_at': '2020-04-01T00:00:00Z'}, - 'tickets': {'updated_at': '2019-06-01T00:00:00Z'}, - 'tickets_deleted': {'updated_at': '2020-05-01T00:00:00Z'}, - 'tickets_spam': {'updated_at': '2020-04-01T00:00:00Z'}} - expected_state_3 = { - **expected_state_1, - "tickets": {"updated_at": "2020-03-16T00:00:00Z"}, - } - - @parameterized.expand([ - ["parent_selected", ["tickets"], ["tickets"], only_parent_response, 5, written_states_1], - ["child_selected", ["conversations"], ["tickets", "conversations"], with_child_response, 4, written_states_2], - ["parent_child_both_selected", ["tickets", "conversations"], ["tickets", "conversations"], with_child_response, 7, written_states_3], - ]) - @mock.patch("singer.write_record") - @mock.patch("singer.write_bookmark") - def test_stream(self, name, selected_streams, streams_to_sync, responses, written_records, written_states, mock_write_bookmark, mock_write_record): - """ - Test that stream is writing records and bookmarks only if selected. - """ - stream = Tickets() - state = {} - client = mock.Mock() - client.base_url = "" - client.request.side_effect = responses - catalog = [ - {"schema":{}, "tap_stream_id": "tickets", "metadata": []}, - {"schema":{}, "tap_stream_id": "conversations", "metadata": []} - ] - - stream.sync_obj(state, self.start_date, client, catalog, selected_streams, streams_to_sync) - - # Verify expected records are written - self.assertEqual(mock_write_record.call_count, written_records) - - # Verify max bookmark is updated for all selected streams - for stream, bookmark in written_states.items(): - mock_write_bookmark.assert_any_call({}, stream, "updated_at", bookmark) - - - @parameterized.expand([ - ["without_state", dict(), expected_state_1, 13], - ["with_parent_state", {"bookmarks": {"tickets": {"updated_at": "2020-03-16T00:00:00Z"}}}, expected_state_2, 10], - ["with_child_state", {"bookmarks": {"conversations": {"updated_at": "2020-03-23T00:00:00Z"}}}, expected_state_1, 8], - ["with_both_state", {"bookmarks": {"tickets": {"updated_at": "2020-03-16T00:00:00Z"}, "conversations": {"updated_at": "2020-03-23T00:00:00Z"}}}, expected_state_3, 5], - ]) - @mock.patch("singer.write_record") - def test_parent_child_both_selected(self, name, state, expected_state, written_records, mock_write_record): - """ - Test parent and child streams both selected in given conditions: - - Without state - - With only parent bookmark - - With only child bookmark - - With both parent and child bookmark - """ - stream = Tickets() - client = mock.Mock() - client.base_url = "" - client.request.side_effect = [ - [{"id": i, "updated_at": f"2020-03-{i}T00:00:00Z"} for i in [11,15,12]], # Tickets Response - [{"id": 10+i, "updated_at": f"2020-03-{i}T00:00:00Z"} for i in [13,24]], # conversations Response - [{"id": 13, "updated_at": "2020-03-01T00:00:00Z", "last_edited_at": "2020-03-02T00:00:00Z"}], # conversations Response - [{"id": 95, "updated_at": "2020-04-01T00:00:00Z"}], # conversations Response - [{"id": 73, "updated_at": "2020-05-01T00:00:00Z"}], # Deleted tickets response - [{"id": 30+i, "updated_at": f"2020-03-{i}T00:00:00Z"}for i in [22,10]], # conversations response - [{"id": 43, "updated_at": "2020-04-01T00:00:00Z"}], # Spam tickets response - [{"id": 50+i, "updated_at": f"2020-03-{i}T00:00:00Z"}for i in [12,25]], # conversations response - ] - catalog = [ - {"schema":{}, "tap_stream_id": "tickets", "metadata": []}, - {"schema":{}, "tap_stream_id": "conversations", "metadata": []} - ] - - stream.sync_obj(state, self.start_date, client, catalog, ["tickets", "conversations"], ["tickets", "conversations"]) - self.assertEqual(mock_write_record.call_count, written_records) - self.assertDictEqual(state, {"bookmarks": expected_state}) - - -class TestSyncTransformDict(unittest.TestCase): - """ - Test `transform_dict` method of stream class. - """ - - stream = Agents() - expected_list_1 = [{"name": "Agency", "value": "Justice League"}, - {"name": "Department", "value": "Superhero"}] - expected_list_2 = [{"key": "Agency", "data": "Justice League"}, - {"key": "Department", "data": "Superhero"}] - expected_list_3 = [{"name": "Agency", "value": "justice league"}, - {"name": "Department", "value": "superhero"}] - @parameterized.expand([ - ["coverting_dict_to_list", {"Agency": "Justice League", "Department": "Superhero"}, expected_list_1, {}], - ["With_custom_keys", {"Agency": "Justice League", "Department": "Superhero"}, expected_list_2, {"key_key":"key", "value_key":"data"}], - ["With_string_value", {"Agency": "Justice League", "Department": "Superhero"}, expected_list_3, {"force_str": True}], - ]) - def test_transform(self, name, dictionary, expected_list, kwargs): - """ - Test that the dictionary is transformed as per given conditions: - - Value is a lowercase string when force_str: True - - Key-Values can be customized by passing in args - """ - returned_list = self.stream.transform_dict(dictionary, **kwargs) - - # Verify returned list is expected - self.assertEqual(returned_list, expected_list) diff --git a/tests/unittests/test_conversations.py b/tests/unittests/test_conversations.py index 947e8c4..0070ca2 100644 --- a/tests/unittests/test_conversations.py +++ b/tests/unittests/test_conversations.py @@ -12,24 +12,27 @@ class TestConversationSync(unittest.TestCase): start_date = "2019-06-01T00:00:00Z" responses_1 = [ - [{"id": "33", "updated_at": "2020-03-01T00:00:00Z"}], # Tickets Response - [{"id": "44", "updated_at": "2020-04-01T00:00:00Z", "last_edited_at": "2020-05-01T00:00:00Z"}], # conversations Response + [{"id": "33", "updated_at": "2020-03-01T00:00:00Z"}], # Tickets Response + [{"id": "44", "updated_at": "2020-04-01T00:00:00Z", + "last_edited_at": "2020-05-01T00:00:00Z"}], # conversations Response [], [] ] responses_2 = [ - [{"id": "33", "updated_at": "2020-03-01T00:00:00Z"}], # Tickets Response - [{"id": "44", "updated_at": "2020-04-01T00:00:00Z", "last_edited_at": None}], # conversations Response + [{"id": "33", "updated_at": "2020-03-01T00:00:00Z"}], # Tickets Response + [{"id": "44", "updated_at": "2020-04-01T00:00:00Z", + "last_edited_at": None}], # conversations Response [], [] ] @parameterized.expand([ + # ["test_name", "responses", "expected_updated_at"] ["with last_edited_at value", responses_1, "2020-05-01T00:00:00Z"], ["with null last_edited_at", responses_2, "2020-04-01T00:00:00Z"], ]) @mock.patch("singer.write_record") @mock.patch("singer.write_bookmark") - def test_stream(self, name, responses, expected_updated_at, mock_write_bookmark, mock_write_record): + def test_stream(self, test_name, responses, expected_updated_at, mock_write_bookmark, mock_write_record): """ Test that the stream is writing the expected record. """ @@ -42,8 +45,8 @@ def test_stream(self, name, responses, expected_updated_at, mock_write_bookmark, # Record with expected `updated_at` value expected_record = {**responses[1][0], "updated_at": expected_updated_at} catalog = [ - {"schema":{}, "tap_stream_id": "tickets", "metadata": []}, - {"schema":{}, "tap_stream_id": "conversations", "metadata": []} + {"schema": {}, "tap_stream_id": "tickets", "metadata": []}, + {"schema": {}, "tap_stream_id": "conversations", "metadata": []} ] stream.sync_obj(state, self.start_date, client, catalog, ["conversations"], ["tickets"]) From d4ed9f1c9cd846aa20d33e5d0e3c457533b3263c Mon Sep 17 00:00:00 2001 From: prijendev Date: Wed, 21 Sep 2022 12:18:56 +0530 Subject: [PATCH 5/5] Resolved pylint errors. --- tap_freshdesk/streams.py | 2 +- tests/unittests/test_conversations.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tap_freshdesk/streams.py b/tap_freshdesk/streams.py index d5d5a52..2d20f7a 100644 --- a/tap_freshdesk/streams.py +++ b/tap_freshdesk/streams.py @@ -93,7 +93,7 @@ def build_url(self, base_url, *args): return base_url + '/api/v2/' + self.path.format(*args) def add_fields_at_1st_level(self, record): - pass + """Adding nested fields at first level.""" def write_records(self, catalog, state, selected_streams, start_date, data, max_bookmark, client, streams_to_sync, child_max_bookmarks, predefined_filter=None): diff --git a/tests/unittests/test_conversations.py b/tests/unittests/test_conversations.py index 0070ca2..b4555f9 100644 --- a/tests/unittests/test_conversations.py +++ b/tests/unittests/test_conversations.py @@ -39,6 +39,7 @@ def test_stream(self, test_name, responses, expected_updated_at, mock_write_book stream = Tickets() state = {} client = mock.Mock() + client.page_size = 100 client.base_url = "" client.request.side_effect = responses