Merge pull request #1 from svinstech/update/remove-anyOf

[DE-2698] Update/remove any of
svinstech · Nov 12, 2022 · e8ec61e · e8ec61e
2 parents 7228b72 + 8c2c2c3
commit e8ec61e
Show file tree

Hide file tree

Showing 8 changed files with 88 additions and 154 deletions.
diff --git a/README.md b/README.md
@@ -84,3 +84,28 @@ In practice, this means that prefacing commits with `feat:` will create a minor
 ### Running a GraphQL query locally
 
 Use the [Anvil Postman collection](https://www.postman.com/useanvil/workspace/anvil/overview).
+
+
+### Caveat
+
+Previously for `CreatedAt`, `UpdatedAt` and some other date attribute, we have 
+```
+ "createdAt": {
+      "anyOf": [
+        {
+          "type": "null"
+        },
+        {
+          "format": "date-time",
+          "type": "string"
+        }
+	]
+    },
+```
+this kind of type for them. Because we're using `target-snowflake` downstream of `tap-anvil` to load this data into our datawarehouse in Snowflake and there is a [known issue](https://github.com/transferwise/pipelinewise-target-snowflake/issues/228) in `target-snowflake` that it will skip `anyOf` type attribute. So we just make this a `string` type like below.
+```
+"type": [
+        "null",
+        "string"
+      ]
+```
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "tap-anvil"
-version = "0.1.5"
+version = "0.1.6"
 description = "A Singer tap for Anvil, a tool for programmatically filling out PDF forms."
 authors = ["Kshitij Aranke <[email protected]>"]
 keywords = ["singer", "tap", "anvil"]

diff --git a/tap_anvil/client.py b/tap_anvil/client.py
@@ -9,7 +9,11 @@
 
 
 class AnvilStream(GraphQLStream):
-    """anvil stream class."""
+    """
+    anvil stream class.
+    note: treat attributes like updatedAt as a string type
+        anyOf type will be ignored by target-snowflake
+    """
 
     url_base = "https://graphql.useanvil.com"
     primary_keys = ["eid"]

diff --git a/tap_anvil/schemas/forges.json b/tap_anvil/schemas/forges.json
@@ -26,14 +26,9 @@
       ]
     },
     "publishedAt": {
-      "anyOf": [
-        {
-          "type": "null"
-        },
-        {
-          "format": "date-time",
-          "type": "string"
-        }
+      "type": [
+        "null",
+        "string"
       ]
     },
     "hasUnpublishedChanges": {
@@ -82,36 +77,21 @@
       ]
     },
     "createdAt": {
-      "anyOf": [
-        {
-          "type": "null"
-        },
-        {
-          "format": "date-time",
-          "type": "string"
-        }
+      "type": [
+        "null",
+        "string"
       ]
     },
     "updatedAt": {
-      "anyOf": [
-        {
-          "type": "null"
-        },
-        {
-          "format": "date-time",
-          "type": "string"
-        }
+      "type": [
+        "null",
+        "string"
       ]
     },
     "archivedAt": {
-      "anyOf": [
-        {
-          "type": "null"
-        },
-        {
-          "format": "date-time",
-          "type": "string"
-        }
+      "type": [
+        "null",
+        "string"
       ]
     },
     "examplePayload": {

diff --git a/tap_anvil/schemas/organizations.json b/tap_anvil/schemas/organizations.json
@@ -77,25 +77,15 @@
       ]
     },
     "createdAt": {
-      "anyOf": [
-        {
-          "type": "null"
-        },
-        {
-          "format": "date-time",
-          "type": "string"
-        }
+      "type": [
+        "null",
+        "string"
       ]
     },
     "updatedAt": {
-      "anyOf": [
-        {
-          "type": "null"
-        },
-        {
-          "format": "date-time",
-          "type": "string"
-        }
+      "type": [
+        "null",
+        "string"
       ]
     },
     "subscribedPlanFeatures": {

diff --git a/tap_anvil/schemas/submissions.json b/tap_anvil/schemas/submissions.json
@@ -83,36 +83,21 @@
       ]
     },
     "createdAt": {
-      "anyOf": [
-        {
-          "type": "null"
-        },
-        {
-          "format": "date-time",
-          "type": "string"
-        }
+      "type": [
+        "null",
+        "string"
       ]
     },
     "updatedAt": {
-      "anyOf": [
-        {
-          "type": "null"
-        },
-        {
-          "format": "date-time",
-          "type": "string"
-        }
+      "type": [
+        "null",
+        "string"
       ]
     },
     "archivedAt": {
-      "anyOf": [
-        {
-          "type": "null"
-        },
-        {
-          "format": "date-time",
-          "type": "string"
-        }
+      "type": [
+        "null",
+        "string"
       ]
     },
     "weldDataEid": {

diff --git a/tap_anvil/schemas/weldDatas.json b/tap_anvil/schemas/weldDatas.json
@@ -113,58 +113,33 @@
       ]
     },
     "createdAt": {
-      "anyOf": [
-        {
-          "type": "null"
-        },
-        {
-          "format": "date-time",
-          "type": "string"
-        }
+      "type": [
+        "null",
+        "string"
       ]
     },
     "updatedAt": {
-      "anyOf": [
-        {
-          "type": "null"
-        },
-        {
-          "format": "date-time",
-          "type": "string"
-        }
+      "type": [
+        "null",
+        "string"
       ]
     },
     "dataUpdatedAt": {
-      "anyOf": [
-        {
-          "type": "null"
-        },
-        {
-          "format": "date-time",
-          "type": "string"
-        }
+      "type": [
+        "null",
+        "string"
       ]
     },
     "expiresAt": {
-      "anyOf": [
-        {
-          "type": "null"
-        },
-        {
-          "format": "date-time",
-          "type": "string"
-        }
+      "type": [
+        "null",
+        "string"
       ]
     },
     "archivedAt": {
-      "anyOf": [
-        {
-          "type": "null"
-        },
-        {
-          "format": "date-time",
-          "type": "string"
-        }
+      "type": [
+        "null",
+        "string"
       ]
     },
     "weldEid": {

diff --git a/tap_anvil/schemas/welds.json b/tap_anvil/schemas/welds.json
@@ -26,14 +26,9 @@
       ]
     },
     "publishedAt": {
-      "anyOf": [
-        {
-          "type": "null"
-        },
-        {
-          "format": "date-time",
-          "type": "string"
-        }
+      "type": [
+        "null",
+        "string"
       ]
     },
     "hasUnpublishedChanges": {
@@ -100,47 +95,27 @@
       ]
     },
     "createdAt": {
-      "anyOf": [
-        {
-          "type": "null"
-        },
-        {
-          "format": "date-time",
-          "type": "string"
-        }
+      "type": [
+        "null",
+        "string"
       ]
     },
     "updatedAt": {
-      "anyOf": [
-        {
-          "type": "null"
-        },
-        {
-          "format": "date-time",
-          "type": "string"
-        }
+      "type": [
+        "null",
+        "string"
       ]
     },
     "archivedAt": {
-      "anyOf": [
-        {
-          "type": "null"
-        },
-        {
-          "format": "date-time",
-          "type": "string"
-        }
+      "type": [
+        "null",
+        "string"
       ]
     },
     "expiresAt": {
-      "anyOf": [
-        {
-          "type": "null"
-        },
-        {
-          "format": "date-time",
-          "type": "string"
-        }
+      "type": [
+        "null",
+        "object"
       ]
     },
     "organizationSlug": {