Fix all_fields_test, break some long lines in other files, fix two as…

…ssertions in payout_trans test

tests/test_all_fields.py

@@ -1,16 +1,14 @@
 import os
+from collections import namedtuple
+from datetime import datetime as dt
+from dateutil.parser import parse
 from pathlib import Path
 from random import random
 from time import sleep, perf_counter
-from datetime import datetime as dt
-from dateutil.parser import parse
-
-from collections import namedtuple
 
-from tap_tester import menagerie, runner, connections, LOGGER
 from base import BaseTapTest
-from utils import \
-    create_object, delete_object, list_all_object, stripe_obj_to_dict
+from tap_tester import menagerie, runner, connections, LOGGER
+from utils import create_object, delete_object, list_all_object, stripe_obj_to_dict
 
 
 # BUG_12478 | https://jira.talendforge.org/browse/TDL-12478
@@ -24,22 +22,28 @@
         'automatic_tax',
         'cancellation_details',
         'default_tax_rates',
+        'discounts',
         'on_behalf_of',
         'payment_settings',
         'pending_update',
         'trial_settings',
     },
     'products': {
         'features',
+        'marketing_features',
     },
     'invoice_items': {
         'price',
     },
     'payouts': {
+        'application_fee',
+        'application_fee_amount',
         'reconciliation_status',
     },
     'charges': set(),
-    'subscription_items': set(),
+    'subscription_items': {
+        'discounts',
+    },
     'plans': set(),
     'invoice_line_items': {
         'margins',
@@ -271,7 +275,9 @@
     'coupons': {
         'times_redeemed' # expect 0, get 1
     },
-    'customers': set(),
+    'customers': {
+        'discount',
+    },
     'subscriptions': set(),
     'products': set(),
     'invoice_items': set(),
@@ -599,6 +605,10 @@ def all_fields_test(self, streams_to_test):
 
                                     elif actual_field_value and field in FICKLE_FIELDS[stream]:
                                         self.assertIsInstance(actual_field_value, type(expected_field_value))
+                                        # if fickle field is a dict at least compare top level keys
+                                        if isinstance(actual_field_value, dict):
+                                            self.assertTrue(set(actual_field_value.keys()).issubset(
+                                                set(expected_field_value.keys())))
 
                                     elif actual_field_value:
                                         raise AssertionError(f"{base_err_msg} Unexpected field is being fickle.")

tests/test_automatic_payout_transactions.py

@@ -1,17 +1,19 @@
-from utils import stripe_obj_to_dict, client, midnight
 from datetime import datetime as dt
 from datetime import timedelta, time
-from tap_tester import runner, connections
+from utils import stripe_obj_to_dict, client, midnight
+
 from base import BaseTapTest
+from tap_tester import runner, connections
+
 
 def get_payouts():
     """
-        Return all the payouts (with pagination), to determine the automatic and non-automatic payouts
+    Return all the payouts (with pagination), to determine the automatic and non-automatic payouts
     """
     # list of all data to return
     four_days_ago = int(dt.combine(dt.today()-timedelta(days=4), time.min).timestamp())
     data = []
-    # Api call of 1st page starting from 4 days ago as there is a lag from the Stripe side to reflect
+    # Api call of 1st page starting from 4 days ago as there is lag from the Stripe side to reflect
     # the automatic payout transactions data
     stripe_obj = client["payouts"].list(limit=100, created={"gte": four_days_ago})
     dict_obj = stripe_obj_to_dict(stripe_obj)
@@ -20,11 +22,13 @@ def get_payouts():
         # add data
         data += dict_obj['data']
     except KeyError:
-        raise Exception("No records for 'Payouts' were replicated, please run 'test_all_fields' before re-running.")
+        raise Exception("No records for 'Payouts' were replicated, please run 'test_all_fields' "
+                        "before re-running.")
 
     # loop over rest of the pages and collect data
     while dict_obj.get("has_more"):
-        stripe_obj = client["payouts"].list(limit=100, created={"gte": four_days_ago}, starting_after=dict_obj.get('data')[-1].get('id'))
+        stripe_obj = client["payouts"].list(limit=100, created={"gte": four_days_ago},
+                                            starting_after=dict_obj.get('data')[-1].get('id'))
         dict_obj = stripe_obj_to_dict(stripe_obj)
         data += dict_obj['data']
 
@@ -33,7 +37,8 @@ def get_payouts():
 
 class AutomaticPayoutTransactionTest(BaseTapTest):
     """
-        Test case to verify that we only collect payout_transactions for payouts containing "automatic" field as "True"
+        Test case to verify that we only collect payout_transactions for payouts containing
+        "automatic" field as "True"
         Prerequisite:
             Run 'test_all_fields' before running this test case.
     """
@@ -60,8 +65,8 @@ def setUpClass(cls):
                 cls.payouts_with_automatic_false.append(record.get("id"))
 
     def test_run(self):
-        # Decreased the start_date for payout_transactions stream as there is a lag from the Stripe side to reflect
-        # the automatic payout transactions data
+        # Decreased the start_date for payout_transactions stream as there is a lag from the Stripe
+        # side to reflect the automatic payout transactions data
         self.start_date = dt.strftime(dt.today() - timedelta(days=4), self.START_DATE_FORMAT)
         conn_id = connections.ensure_connection(self, original_properties=False)
 
@@ -89,21 +94,23 @@ def test_run(self):
                                msg="Data isn't set up to be able to test full sync")
 
             # get records
-            records = [message.get("data") for message in first_sync_records.get(stream).get("messages") if message["action"] == "upsert"]
+            records = [message.get("data") for message
+                       in first_sync_records.get(stream).get("messages")
+                       if message["action"] == "upsert"]
 
             # collect payout ids for all the payout transaction records
             payout_transaction_payout_ids = set()
             for record in records:
                 payout_transaction_payout_ids.add(record.get("payout_id"))
 
             # verify that data exists for payouts with "automatic" field as "True" and "False"
-            self.assertTrue(self.payouts_with_automatic_true is not None)
-            self.assertTrue(self.payouts_with_automatic_false is not None)
+            self.assertGreater(len(self.payouts_with_automatic_true), 0)
+            self.assertGreater(len(self.payouts_with_automatic_false), 0)
 
             # loop over all the payout ids from the payout transactions to verify
             # that we collected "payout transactions" of automatic payouts only
             for id in payout_transaction_payout_ids:
-                # verify that we collect payout transaction record for payout containing "automatic": True
+                # verify payout transaction record collected for payout containing "automatic": True
                 self.assertTrue(id in self.payouts_with_automatic_true)
-                # verify that we do not collect payout transaction record for payout containing "automatic": False
+                # verify payout transaction rec NOT collected for payout with "automatic": False
                 self.assertTrue(id not in self.payouts_with_automatic_false)

tests/test_create_object.py

@@ -25,12 +25,14 @@ def test_run(self):
         self.conn_id = conn_id
 
         streams_to_create = {
-            "balance_transactions",  # should be created implicity with a create in the payouts or charges streams
+            "balance_transactions",  # should be created implicity with create in payouts or charges
             "charges",
             "coupons",
             "customers",
             "invoice_items",
-            "invoice_line_items", # this is created implicity by invoices, it just creates another invoice TODO get this outa here
+            # invoice_line_items are created implicity by invoices, this creates another invoice
+            # TODO update test to remove invoice_line_items from here
+            "invoice_line_items",
             "invoices", # this will create an invoice_item
             "payouts",
             "plans",
@@ -41,7 +43,7 @@ def test_run(self):
          }
 
         missing_streams_to_create = {
-            "disputes",  # can be created by simulating a dispute transaction with a specific card number
+            "disputes",  # create by simulating a dispute transaction with a specific card number
             # no way to create directly, see: https://stripe.com/docs/testing#disputes
             "payout_transactions",  # BUG_9703 | https://jira.talendforge.org/browse/TDL-9703
             # depends on payouts and transactions
@@ -132,7 +134,7 @@ def test_run(self):
                                 f"new_id: {new_objects[stream]['id']}")
                     self.assertTrue(new_objects[stream]['id'] in null_date_invoices)
                     if new_objects[stream]['id'] not in masking_invoices:
-                        LOGGER.warn(f"########## Previous error scenario detected (un-masked failure) ##########")
+                        LOGGER.warn(f"### Previous error scenario detected (un-masked failure) ###")
                 # TODO END DEBUG
 
                 # verify the new object is in the list of created objects