From 4b2cfcee1889404012b8525c28c5d3a65b2234a7 Mon Sep 17 00:00:00 2001 From: treeforest456 Date: Fri, 11 Nov 2022 11:33:44 -0800 Subject: [PATCH 1/4] remove anyOf type which is skipped by target-snowflake --- tap_anvil/schemas/forges.json | 44 ++++++---------------- tap_anvil/schemas/organizations.json | 22 +++-------- tap_anvil/schemas/submissions.json | 33 +++++------------ tap_anvil/schemas/weldDatas.json | 55 ++++++++-------------------- tap_anvil/schemas/welds.json | 44 ++++++---------------- 5 files changed, 54 insertions(+), 144 deletions(-) diff --git a/tap_anvil/schemas/forges.json b/tap_anvil/schemas/forges.json index 1496649..f906871 100644 --- a/tap_anvil/schemas/forges.json +++ b/tap_anvil/schemas/forges.json @@ -26,14 +26,9 @@ ] }, "publishedAt": { - "anyOf": [ - { - "type": "null" - }, - { - "format": "date-time", - "type": "string" - } + "type": [ + "null", + "object" ] }, "hasUnpublishedChanges": { @@ -82,36 +77,21 @@ ] }, "createdAt": { - "anyOf": [ - { - "type": "null" - }, - { - "format": "date-time", - "type": "string" - } + "type": [ + "null", + "object" ] }, "updatedAt": { - "anyOf": [ - { - "type": "null" - }, - { - "format": "date-time", - "type": "string" - } + "type": [ + "null", + "object" ] }, "archivedAt": { - "anyOf": [ - { - "type": "null" - }, - { - "format": "date-time", - "type": "string" - } + "type": [ + "null", + "object" ] }, "examplePayload": { diff --git a/tap_anvil/schemas/organizations.json b/tap_anvil/schemas/organizations.json index 82a8660..0b9339b 100644 --- a/tap_anvil/schemas/organizations.json +++ b/tap_anvil/schemas/organizations.json @@ -77,25 +77,15 @@ ] }, "createdAt": { - "anyOf": [ - { - "type": "null" - }, - { - "format": "date-time", - "type": "string" - } + "type": [ + "null", + "object" ] }, "updatedAt": { - "anyOf": [ - { - "type": "null" - }, - { - "format": "date-time", - "type": "string" - } + "type": [ + "null", + "object" ] }, "subscribedPlanFeatures": { diff --git a/tap_anvil/schemas/submissions.json b/tap_anvil/schemas/submissions.json index 6709279..449a919 100644 --- a/tap_anvil/schemas/submissions.json +++ b/tap_anvil/schemas/submissions.json @@ -83,36 +83,21 @@ ] }, "createdAt": { - "anyOf": [ - { - "type": "null" - }, - { - "format": "date-time", - "type": "string" - } + "type": [ + "null", + "object" ] }, "updatedAt": { - "anyOf": [ - { - "type": "null" - }, - { - "format": "date-time", - "type": "string" - } + "type": [ + "null", + "object" ] }, "archivedAt": { - "anyOf": [ - { - "type": "null" - }, - { - "format": "date-time", - "type": "string" - } + "type": [ + "null", + "object" ] }, "weldDataEid": { diff --git a/tap_anvil/schemas/weldDatas.json b/tap_anvil/schemas/weldDatas.json index 8aba2ef..447a499 100644 --- a/tap_anvil/schemas/weldDatas.json +++ b/tap_anvil/schemas/weldDatas.json @@ -113,58 +113,33 @@ ] }, "createdAt": { - "anyOf": [ - { - "type": "null" - }, - { - "format": "date-time", - "type": "string" - } + "type": [ + "null", + "object" ] }, "updatedAt": { - "anyOf": [ - { - "type": "null" - }, - { - "format": "date-time", - "type": "string" - } + "type": [ + "null", + "object" ] }, "dataUpdatedAt": { - "anyOf": [ - { - "type": "null" - }, - { - "format": "date-time", - "type": "string" - } + "type": [ + "null", + "object" ] }, "expiresAt": { - "anyOf": [ - { - "type": "null" - }, - { - "format": "date-time", - "type": "string" - } + "type": [ + "null", + "object" ] }, "archivedAt": { - "anyOf": [ - { - "type": "null" - }, - { - "format": "date-time", - "type": "string" - } + "type": [ + "null", + "object" ] }, "weldEid": { diff --git a/tap_anvil/schemas/welds.json b/tap_anvil/schemas/welds.json index 23c79ad..d035bc1 100644 --- a/tap_anvil/schemas/welds.json +++ b/tap_anvil/schemas/welds.json @@ -100,47 +100,27 @@ ] }, "createdAt": { - "anyOf": [ - { - "type": "null" - }, - { - "format": "date-time", - "type": "string" - } + "type": [ + "null", + "object" ] }, "updatedAt": { - "anyOf": [ - { - "type": "null" - }, - { - "format": "date-time", - "type": "string" - } + "type": [ + "null", + "object" ] }, "archivedAt": { - "anyOf": [ - { - "type": "null" - }, - { - "format": "date-time", - "type": "string" - } + "type": [ + "null", + "object" ] }, "expiresAt": { - "anyOf": [ - { - "type": "null" - }, - { - "format": "date-time", - "type": "string" - } + "type": [ + "null", + "object" ] }, "organizationSlug": { From d5390f8fd1b828c6c6db9125b173841305630a0c Mon Sep 17 00:00:00 2001 From: treeforest456 Date: Fri, 11 Nov 2022 11:52:28 -0800 Subject: [PATCH 2/4] string not object --- tap_anvil/schemas/forges.json | 8 ++++---- tap_anvil/schemas/organizations.json | 4 ++-- tap_anvil/schemas/submissions.json | 6 +++--- tap_anvil/schemas/weldDatas.json | 10 +++++----- tap_anvil/schemas/welds.json | 17 ++++++----------- 5 files changed, 20 insertions(+), 25 deletions(-) diff --git a/tap_anvil/schemas/forges.json b/tap_anvil/schemas/forges.json index f906871..5700563 100644 --- a/tap_anvil/schemas/forges.json +++ b/tap_anvil/schemas/forges.json @@ -28,7 +28,7 @@ "publishedAt": { "type": [ "null", - "object" + "string" ] }, "hasUnpublishedChanges": { @@ -79,19 +79,19 @@ "createdAt": { "type": [ "null", - "object" + "string" ] }, "updatedAt": { "type": [ "null", - "object" + "string" ] }, "archivedAt": { "type": [ "null", - "object" + "string" ] }, "examplePayload": { diff --git a/tap_anvil/schemas/organizations.json b/tap_anvil/schemas/organizations.json index 0b9339b..ce5ea0e 100644 --- a/tap_anvil/schemas/organizations.json +++ b/tap_anvil/schemas/organizations.json @@ -79,13 +79,13 @@ "createdAt": { "type": [ "null", - "object" + "string" ] }, "updatedAt": { "type": [ "null", - "object" + "string" ] }, "subscribedPlanFeatures": { diff --git a/tap_anvil/schemas/submissions.json b/tap_anvil/schemas/submissions.json index 449a919..aebb320 100644 --- a/tap_anvil/schemas/submissions.json +++ b/tap_anvil/schemas/submissions.json @@ -85,19 +85,19 @@ "createdAt": { "type": [ "null", - "object" + "string" ] }, "updatedAt": { "type": [ "null", - "object" + "string" ] }, "archivedAt": { "type": [ "null", - "object" + "string" ] }, "weldDataEid": { diff --git a/tap_anvil/schemas/weldDatas.json b/tap_anvil/schemas/weldDatas.json index 447a499..fdfd61d 100644 --- a/tap_anvil/schemas/weldDatas.json +++ b/tap_anvil/schemas/weldDatas.json @@ -115,31 +115,31 @@ "createdAt": { "type": [ "null", - "object" + "string" ] }, "updatedAt": { "type": [ "null", - "object" + "string" ] }, "dataUpdatedAt": { "type": [ "null", - "object" + "string" ] }, "expiresAt": { "type": [ "null", - "object" + "string" ] }, "archivedAt": { "type": [ "null", - "object" + "string" ] }, "weldEid": { diff --git a/tap_anvil/schemas/welds.json b/tap_anvil/schemas/welds.json index d035bc1..784ff80 100644 --- a/tap_anvil/schemas/welds.json +++ b/tap_anvil/schemas/welds.json @@ -26,14 +26,9 @@ ] }, "publishedAt": { - "anyOf": [ - { - "type": "null" - }, - { - "format": "date-time", - "type": "string" - } + "type": [ + "null", + "string" ] }, "hasUnpublishedChanges": { @@ -102,19 +97,19 @@ "createdAt": { "type": [ "null", - "object" + "string" ] }, "updatedAt": { "type": [ "null", - "object" + "string" ] }, "archivedAt": { "type": [ "null", - "object" + "string" ] }, "expiresAt": { From 5db2eb12ef6d14a553ebe7d0e5f425e34a5e523c Mon Sep 17 00:00:00 2001 From: treeforest456 Date: Fri, 11 Nov 2022 13:15:15 -0800 Subject: [PATCH 3/4] add comments in clients.py and notes in readme --- README.md | 25 +++++++++++++++++++++++++ tap_anvil/client.py | 6 +++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f8ebf50..dac4b45 100644 --- a/README.md +++ b/README.md @@ -84,3 +84,28 @@ In practice, this means that prefacing commits with `feat:` will create a minor ### Running a GraphQL query locally Use the [Anvil Postman collection](https://www.postman.com/useanvil/workspace/anvil/overview). + + +### Caveat + +Previously for `CreatedAt`, `UpdatedAt` and some other date attribute, we have +``` + "createdAt": { + "anyOf": [ + { + "type": "null" + }, + { + "format": "date-time", + "type": "string" + } + ] + }, +``` +this kind of type for them. Because we're using `target-snowflake` downstream of `tap-anvil` to load this data into our datawarehouse in Snowflake and there is a [known issue](https://github.com/transferwise/pipelinewise-target-snowflake/issues/228) in `target-snowflake` that it will skip `anyOf` type attribute. So we just make this a `string` type like below. +``` +"type": [ + "null", + "string" + ] +``` diff --git a/tap_anvil/client.py b/tap_anvil/client.py index f39c669..5078048 100644 --- a/tap_anvil/client.py +++ b/tap_anvil/client.py @@ -9,7 +9,11 @@ class AnvilStream(GraphQLStream): - """anvil stream class.""" + """ + anvil stream class. + note: treat attributes like updatedAt as a string type + anyOf type will be ignored by target-snowflake + """ url_base = "https://graphql.useanvil.com" primary_keys = ["eid"] From 8c2c2c3daa9f7cd226f95121a9f4ab7c30ecfeb5 Mon Sep 17 00:00:00 2001 From: treeforest456 Date: Fri, 11 Nov 2022 13:23:59 -0800 Subject: [PATCH 4/4] update version number to 0.1.6? --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9ada6e1..422c3a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tap-anvil" -version = "0.1.5" +version = "0.1.6" description = "A Singer tap for Anvil, a tool for programmatically filling out PDF forms." authors = ["Kshitij Aranke "] keywords = ["singer", "tap", "anvil"]