Fix: request selected properties for a given object only (#94)

* request selected properties only

* selected properties for other streams

* remove logging

* get_selected_properties client method

* list comprehension

* PR comments

tap_hubspot/client.py

@@ -86,6 +86,13 @@ def get_url_params(
             params["after"] = next_page_token
         params["limit"] = 100
         return params
+
+    def get_selected_properties(self) -> List[dict]:
+        selected_properties = [
+            key[-1] for key, value in self.metadata.items()
+            if value.selected and len(key) > 0
+            ]
+        return list(set(self.properties).intersection(selected_properties))
 
     def prepare_request_payload(
         self, context: Optional[dict], next_page_token: Optional[Any]

tap_hubspot/streams.py

@@ -20,8 +20,9 @@ class MeetingsStream(HubspotStream):
     def get_url_params(
         self, context: Optional[dict], next_page_token: Optional[Any]
     ) -> Dict[str, Any]:
+        selected_properties = self.get_selected_properties()
         params = super().get_url_params(context, next_page_token)
-        params["properties"] = ",".join(self.properties)
+        params["properties"] = ",".join(selected_properties)
         return params
 
     @property
@@ -39,8 +40,9 @@ class CallsStream(HubspotStream):
     def get_url_params(
         self, context: Optional[dict], next_page_token: Optional[Any]
     ) -> Dict[str, Any]:
+        selected_properties = self.get_selected_properties()
         params = super().get_url_params(context, next_page_token)
-        params["properties"] = ",".join(self.properties)
+        params["properties"] = ",".join(selected_properties)
         return params
 
     @property
@@ -73,11 +75,13 @@ class CompaniesStream(HubspotStream):
     primary_keys = ["id"]
     partitions = [{"archived": True}, {"archived": False}]
 
+
     def get_url_params(
         self, context: Optional[dict], next_page_token: Optional[Any]
     ) -> Dict[str, Any]:
+        selected_properties = self.get_selected_properties()
         params = super().get_url_params(context, next_page_token)
-        params["properties"] = ",".join(self.properties)
+        params["properties"] = ",".join(selected_properties)
         params["archived"] = context["archived"]
         params["associations"] = ",".join(HUBSPOT_OBJECTS)
         return params
@@ -103,8 +107,9 @@ class DealsStream(HubspotStream):
     def get_url_params(
         self, context: Optional[dict], next_page_token: Optional[Any]
     ) -> Dict[str, Any]:
+        selected_properties = self.get_selected_properties()
         params = super().get_url_params(context, next_page_token)
-        params["properties"] = ",".join(self.properties)
+        params["properties"] = ",".join(selected_properties)
         params["archived"] = context["archived"]
         params["associations"] = ",".join(HUBSPOT_OBJECTS)
         return params
@@ -134,8 +139,9 @@ class ContactsStream(HubspotStream):
     def get_url_params(
         self, context: Optional[dict], next_page_token: Optional[Any]
     ) -> Dict[str, Any]:
+        selected_properties = self.get_selected_properties()
         params = super().get_url_params(context, next_page_token)
-        params["properties"] = ",".join(self.properties)
+        params["properties"] = ",".join(selected_properties)
         params["archived"] = context["archived"]
         params["associations"] = ",".join(HUBSPOT_OBJECTS)
         return params
@@ -383,8 +389,9 @@ class QuotesStream(HubspotStream):
     def get_url_params(
         self, context: Optional[dict], next_page_token: Optional[Any]
     ) -> Dict[str, Any]:
+        selected_properties = self.get_selected_properties()
         params = super().get_url_params(context, next_page_token)
-        params["properties"] = ",".join(self.properties)
+        params["properties"] = ",".join(selected_properties)
         params["archived"] = context["archived"]
         params["associations"] = ",".join(HUBSPOT_OBJECTS)
         return params
@@ -404,8 +411,9 @@ class LineItemsStream(HubspotStream):
     def get_url_params(
         self, context: Optional[dict], next_page_token: Optional[Any]
     ) -> Dict[str, Any]:
+        selected_properties = self.get_selected_properties()
         params = super().get_url_params(context, next_page_token)
-        params["properties"] = ",".join(self.properties)
+        params["properties"] = ",".join(selected_properties)
         params["archived"] = context["archived"]
         params["associations"] = ",".join(HUBSPOT_OBJECTS)
         return params