diff --git a/meltano.yml b/meltano.yml index 045577f..a7f6a62 100644 --- a/meltano.yml +++ b/meltano.yml @@ -20,6 +20,8 @@ plugins: sensitive: true - name: custom_streams kind: array + config: + custom_streams: [{"name": "container_vulnerabilities", "primary_keys": ["id"], "query": "{\"type\": \"object_set\", \"models\": [\"Container\"], \"keys\": [\"Container\"], \"with\": {\"operator\": \"and\", \"type\": \"operation\", \"values\": [{\"operator\": \"and\", \"type\": \"operation\", \"values\": [{\"type\": \"object_set\", \"keys\": [\"Vulnerabilities\"], \"models\": [\"Vulnerability\"], \"operator\": \"has\", \"negate\": false, \"disabled\": false, \"with\": {\"type\": \"operation\", \"negate\": false, \"disabled\": false, \"operator\": \"and\", \"values\": [{\"type\": \"object\", \"keys\": [\"CVE\"], \"models\": [\"CVE\"], \"operator\": \"has\", \"negate\": false, \"disabled\": false, \"with\": {\"type\": \"operation\", \"negate\": false, \"disabled\": false, \"operator\": \"and\", \"values\": [{\"type\": \"str\", \"key\": \"Cvss3Severity\", \"operator\": \"in\", \"values\": [\"CRITICAL\", \"HIGH\", \"MEDIUM\"]}]}}, {\"key\": \"FixAvailable\", \"type\": \"str\", \"values\": [\"Yes\", \"Extended\"], \"operator\": \"in\"}, {\"keys\": [\"CVEVendorData\"], \"type\": \"object_set\", \"with\": {\"type\": \"operation\", \"values\": [], \"operator\": \"and\"}, \"models\": [\"CVEDescription\"], \"operator\": \"has\"}]}}, {\"type\": \"operation\", \"negate\": false, \"disabled\": false, \"operator\": \"and\", \"values\": [{\"keys\": [\"AssetGroup\"], \"type\": \"object\", \"models\": [\"Group\"], \"negate\": false, \"disabled\": false, \"operator\": \"has\", \"with\": {\"type\": \"operation\", \"negate\": false, \"values\": [{\"key\": \"ClusterName\", \"type\": \"str\", \"negate\": false, \"values\": [\"prod-dazbogf2bf\"], \"operator\": \"containing\"}], \"disabled\": false, \"operator\": \"or\"}}, {\"type\": \"operation\", \"operator\": \"and\", \"values\": [{\"keys\": [\"CloudAccount\"], \"type\": \"object_set\", \"models\": [\"CloudAccount\"], \"operator\": \"has\", \"with\": {\"type\": \"operation\", \"operator\": \"and\", \"values\": [{\"key\": \"Name\", \"type\": \"str\", \"operator\": \"in\", \"values\": [\"AWS - 345874614325 - Cloudservicesprod\", \"AWS - 821209223267 - Hosting Prod\"]}]}}]}]}]}]}}"}] loaders: - name: target-jsonl variant: andyh1203 diff --git a/pyproject.toml b/pyproject.toml index 428b486..64576ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "acquia-tap-orca" -version = "0.0.1" +version = "0.0.2" description = "`tap-orca` is a Singer tap for Orca, built with the Meltano Singer SDK." readme = "README.md" authors = ["Josh Lloyd "] diff --git a/tap_orca/tap.py b/tap_orca/tap.py index 5e164e6..ab59ce1 100644 --- a/tap_orca/tap.py +++ b/tap_orca/tap.py @@ -97,27 +97,28 @@ def discover_schema(self, query: str) -> dict: prepared_request = stream.prepare_request({}, next_page_token=0) resp = decorated_request(prepared_request, {}) records = iter(stream.parse_response(resp)) - first_record = next(records) + first_record = next(records, None) # accept any type for each key in the first record - for key in first_record.keys(): - schema.append( - th.Property( - key, - th.StringType(), - # th.CustomType(jsonschema_type_dict={ - # "anyOf": [ - # {"type": "null"}, - # {"type": "object"}, - # {"type": "array"}, - # {"type": "string"}, - # {"type": "number"}, - # # {"type": "boolean"}, # causes all values to be a boolean - # {"type": "integer"}, - # ] - # }) - ), - ) + if first_record: + for key in first_record.keys(): + schema.append( + th.Property( + key, + th.StringType(), + # th.CustomType(jsonschema_type_dict={ + # "anyOf": [ + # {"type": "null"}, + # {"type": "object"}, + # {"type": "array"}, + # {"type": "string"}, + # {"type": "number"}, + # # {"type": "boolean"}, # causes all values to be a boolean + # {"type": "integer"}, + # ] + # }) + ), + ) self.logger.info("Schema discovery complete.") return schema.to_dict()