fixed null first record error

meltano.yml

@@ -20,6 +20,8 @@ plugins:
       sensitive: true
     - name: custom_streams
       kind: array
+    config:
+      custom_streams: [{"name": "container_vulnerabilities", "primary_keys": ["id"], "query": "{\"type\": \"object_set\", \"models\": [\"Container\"], \"keys\": [\"Container\"], \"with\": {\"operator\": \"and\", \"type\": \"operation\", \"values\": [{\"operator\": \"and\", \"type\": \"operation\", \"values\": [{\"type\": \"object_set\", \"keys\": [\"Vulnerabilities\"], \"models\": [\"Vulnerability\"], \"operator\": \"has\", \"negate\": false, \"disabled\": false, \"with\": {\"type\": \"operation\", \"negate\": false, \"disabled\": false, \"operator\": \"and\", \"values\": [{\"type\": \"object\", \"keys\": [\"CVE\"], \"models\": [\"CVE\"], \"operator\": \"has\", \"negate\": false, \"disabled\": false, \"with\": {\"type\": \"operation\", \"negate\": false, \"disabled\": false, \"operator\": \"and\", \"values\": [{\"type\": \"str\", \"key\": \"Cvss3Severity\", \"operator\": \"in\", \"values\": [\"CRITICAL\", \"HIGH\", \"MEDIUM\"]}]}}, {\"key\": \"FixAvailable\", \"type\": \"str\", \"values\": [\"Yes\", \"Extended\"], \"operator\": \"in\"}, {\"keys\": [\"CVEVendorData\"], \"type\": \"object_set\", \"with\": {\"type\": \"operation\", \"values\": [], \"operator\": \"and\"}, \"models\": [\"CVEDescription\"], \"operator\": \"has\"}]}}, {\"type\": \"operation\", \"negate\": false, \"disabled\": false, \"operator\": \"and\", \"values\": [{\"keys\": [\"AssetGroup\"], \"type\": \"object\", \"models\": [\"Group\"], \"negate\": false, \"disabled\": false, \"operator\": \"has\", \"with\": {\"type\": \"operation\", \"negate\": false, \"values\": [{\"key\": \"ClusterName\", \"type\": \"str\", \"negate\": false, \"values\": [\"prod-dazbogf2bf\"], \"operator\": \"containing\"}], \"disabled\": false, \"operator\": \"or\"}}, {\"type\": \"operation\", \"operator\": \"and\", \"values\": [{\"keys\": [\"CloudAccount\"], \"type\": \"object_set\", \"models\": [\"CloudAccount\"], \"operator\": \"has\", \"with\": {\"type\": \"operation\", \"operator\": \"and\", \"values\": [{\"key\": \"Name\", \"type\": \"str\", \"operator\": \"in\", \"values\": [\"AWS - 345874614325 - Cloudservicesprod\", \"AWS - 821209223267 - Hosting Prod\"]}]}}]}]}]}]}}"}]
   loaders:
   - name: target-jsonl
     variant: andyh1203

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "acquia-tap-orca"
-version = "0.0.1"
+version = "0.0.2"
 description = "`tap-orca` is a Singer tap for Orca, built with the Meltano Singer SDK."
 readme = "README.md"
 authors = ["Josh Lloyd <[email protected]>"]

tap_orca/tap.py

@@ -97,27 +97,28 @@ def discover_schema(self, query: str) -> dict:
         prepared_request = stream.prepare_request({}, next_page_token=0)
         resp = decorated_request(prepared_request, {})
         records = iter(stream.parse_response(resp))
-        first_record = next(records)
+        first_record = next(records, None)
 
         # accept any type for each key in the first record
-        for key in first_record.keys():
-            schema.append(
-                th.Property(
-                    key,
-                    th.StringType(),
-                    # th.CustomType(jsonschema_type_dict={
-                    #     "anyOf": [
-                    #         {"type": "null"},
-                    #         {"type": "object"},
-                    #         {"type": "array"},
-                    #         {"type": "string"},
-                    #         {"type": "number"},
-                    #         # {"type": "boolean"},  # causes all values to be a boolean
-                    #         {"type": "integer"},
-                    #     ]
-                    # })
-                ),
-            )
+        if first_record:
+            for key in first_record.keys():
+                schema.append(
+                    th.Property(
+                        key,
+                        th.StringType(),
+                        # th.CustomType(jsonschema_type_dict={
+                        #     "anyOf": [
+                        #         {"type": "null"},
+                        #         {"type": "object"},
+                        #         {"type": "array"},
+                        #         {"type": "string"},
+                        #         {"type": "number"},
+                        #         # {"type": "boolean"},  # causes all values to be a boolean
+                        #         {"type": "integer"},
+                        #     ]
+                        # })
+                    ),
+                )
         self.logger.info("Schema discovery complete.")
         return schema.to_dict()