added primary key option

meltano.yml

@@ -17,9 +17,13 @@ plugins:
     - stream-maps
     settings:
     - name: api_token
-      kind: password
+      sensitive: true
     - name: custom_streams
       kind: array
+    config:
+      custom_streams:
+      - name: "example_stream_name"
+        query: '{"models": ["Inventory"], "type": "object_set", "with": {"operator": "and", "type": "operation", "values": [{"operator": "or", "type": "operation", "values": [{"key": "NewSubCategory", "values": ["Exposure"], "type": "str", "operator": "in"}, {"key": "Type", "values": ["Container"], "type": "str", "operator": "in"}]}, {"type": "operation", "values": [{"keys": ["Vulnerabilities"], "type": "object_set", "with": {"type": "operation", "negate": false, "values": [{"keys": ["CVE"], "type": "object", "with": {"type": "operation", "negate": false, "values": [{"key": "Cvss3Severity", "type": "str", "values": ["HIGH", "MEDIUM", "CRITICAL"], "operator": "in"}], "disabled": false, "operator": "and"}, "models": ["CVE"], "negate": false, "disabled": false, "operator": "has"}, {"key": "FixAvailable", "type": "str", "values": ["Yes", "Extended"], "operator": "in"}, {"keys": ["CVEVendorData"], "type": "object_set", "with": {"type": "operation", "values": [], "operator": "and"}, "models": ["CVEDescription"], "operator": "has"}], "disabled": false, "operator": "and"}, "models": ["Vulnerability"], "negate": false, "disabled": false, "operator": "has"}, {"type": "operation", "negate": false, "values": [{"keys": ["AssetGroup"], "type": "object", "with": {"type": "operation", "negate": false, "values": [{"key": "ClusterName", "type": "str", "negate": false, "values": ["prod-wario"], "disabled": false, "operator": "containing"}], "disabled": false, "operator": "or"}, "models": ["Group"], "negate": false, "disabled": false, "operator": "has"}, {"type": "operation", "values": [{"keys": ["CloudAccount"], "type": "object_set", "with": {"type": "operation", "values": [{"key": "Name", "type": "str", "values": ["AWS - 345874614325 - Cloudservicesprod", "AWS - 821209223267 - Hosting Prod"], "operator": "in"}], "operator": "and"}, "models": ["CloudAccount"], "operator": "has"}], "operator": "and"}], "disabled": false, "operator": "and"}], "operator": "and"}]}}'
   loaders:
   - name: target-jsonl
     variant: andyh1203

pyproject.toml

@@ -3,7 +3,7 @@ name = "acquia-tap-orca"
 version = "0.0.1"
 description = "`tap-orca` is a Singer tap for Orca, built with the Meltano Singer SDK."
 readme = "README.md"
-authors = ["Josh <[email protected]>"]
+authors = ["Josh Lloyd <[email protected]>"]
 keywords = [
     "ELT",
     "Orca",

tap_orca/streams.py

@@ -23,7 +23,7 @@ def __init__(
         name: str,
         query: dict | str,
         schema: dict,
-        primary_keys: t.ClassVar[list[str]] = ["id"],
+        primary_keys: t.ClassVar[list[str]] = [],
         replication_key=None,
         **kwargs
     ):
@@ -32,6 +32,7 @@ def __init__(
         self.custom_query = query
         self.primary_keys = primary_keys
         self.replication_key = replication_key
+        self.primary_keys = primary_keys
 
     @property
     def query(self) -> dict:

tap_orca/tap.py

@@ -27,6 +27,12 @@ class TapOrca(Tap):
             required=True,
             description="The query to run against the API."
         ),
+        th.Property(
+            "primary_keys",
+            th.ArrayType(th.StringType),
+            required=False,
+            description="The primary keys for the custom stream."
+        )
     )
 
     config_jsonschema = th.PropertiesList(
@@ -58,6 +64,7 @@ def discover_streams(self) -> list[OrcaStream]:
                 name=custom_stream.get("name"),
                 query=custom_stream.get("query"),
                 schema=self.discover_schema(custom_stream.get("query")),
+                primary_keys=custom_stream.get("primary_keys"),
             ))
         return streams