From d713ddde130306915586510a36638987fe6f1048 Mon Sep 17 00:00:00 2001 From: jlloyd-widen <82222659+jlloyd-widen@users.noreply.github.com> Date: Tue, 6 Sep 2022 06:53:15 -0600 Subject: [PATCH] Rm oauth (#32) * updated tap to use oauth, got pytests working * added lists and list_contacts streams * fixed primary key for contact_lists_v1 stream * fixed pagination for the lists and list contacts * fixed a paging bug to compensate for Hubspot's error on the forms endpoint, added missing fields to schema for forms endpoint * added formType param to forms queries * fixed pytests and added extra retry status codes * added new exception type to request_decorator * removed the oauth capabilities and hardened the schema for forms * cleaned up some unnecessary changes and comments * removed flattening in meltano.yml Co-authored-by: Josh Lloyd Co-authored-by: Josh Lloyd --- .gitignore | 1 + meltano.yml | 3 + poetry.lock | 354 ++++++------------ pyproject.toml | 3 +- tap_hubspot/client.py | 47 ++- tap_hubspot/marketing_streams.py | 76 +++- tap_hubspot/schemas/contacts.json | 4 + tap_hubspot/schemas/list_contacts_v1.json | 70 ++++ tap_hubspot/schemas/lists_v1.json | 84 +++++ tap_hubspot/schemas/marketing/Forms.py | 75 +++- tap_hubspot/schemas/marketing/ListContacts.py | 30 ++ tap_hubspot/schemas/marketing/Lists.py | 39 ++ tap_hubspot/tap.py | 5 +- tap_hubspot/tests/test_core.py | 50 ++- 14 files changed, 556 insertions(+), 285 deletions(-) create mode 100644 tap_hubspot/schemas/list_contacts_v1.json create mode 100644 tap_hubspot/schemas/lists_v1.json create mode 100644 tap_hubspot/schemas/marketing/ListContacts.py create mode 100644 tap_hubspot/schemas/marketing/Lists.py diff --git a/.gitignore b/.gitignore index 57f8984..a03328f 100644 --- a/.gitignore +++ b/.gitignore @@ -114,3 +114,4 @@ catalog.json # Custom Debug *.vscode +*.idea diff --git a/meltano.yml b/meltano.yml index f7256bc..9319961 100644 --- a/meltano.yml +++ b/meltano.yml @@ -13,6 +13,9 @@ plugins: settings: - name: access_token kind: password + - name: flattening_enabled + kind: boolean + - name: flattening_max_depth - name: start_date value: '2010-01-01T00:00:00Z' config: diff --git a/poetry.lock b/poetry.lock index ed2033c..a3abd21 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,6 +1,6 @@ [[package]] name = "atomicwrites" -version = "1.4.0" +version = "1.4.1" description = "Atomic file writes." category = "main" optional = false @@ -30,42 +30,38 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "black" -version = "21.12b0" +version = "22.6.0" description = "The uncompromising code formatter." category = "dev" optional = false python-versions = ">=3.6.2" [package.dependencies] -click = ">=7.1.2" +click = ">=8.0.0" mypy-extensions = ">=0.4.3" -pathspec = ">=0.9.0,<1" +pathspec = ">=0.9.0" platformdirs = ">=2" -tomli = ">=0.2.6,<2.0.0" +tomli = {version = ">=1.1.0", markers = "python_full_version < \"3.11.0a7\""} typed-ast = {version = ">=1.4.2", markers = "python_version < \"3.8\" and implementation_name == \"cpython\""} -typing-extensions = [ - {version = ">=3.10.0.0", markers = "python_version < \"3.10\""}, - {version = "!=3.10.0.1", markers = "python_version >= \"3.10\""}, -] +typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} [package.extras] colorama = ["colorama (>=0.4.3)"] d = ["aiohttp (>=3.7.4)"] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] -python2 = ["typed-ast (>=1.4.3)"] uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "certifi" -version = "2021.10.8" +version = "2022.6.15" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false -python-versions = "*" +python-versions = ">=3.6" [[package]] name = "cffi" -version = "1.15.0" +version = "1.15.1" description = "Foreign Function Interface for Python calling C code." category = "main" optional = false @@ -76,11 +72,11 @@ pycparser = "*" [[package]] name = "charset-normalizer" -version = "2.0.12" +version = "2.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." category = "main" optional = false -python-versions = ">=3.5.0" +python-versions = ">=3.6.0" [package.extras] unicode_backport = ["unicodedata2"] @@ -95,7 +91,7 @@ python-versions = "*" [[package]] name = "click" -version = "8.1.2" +version = "8.1.3" description = "Composable command line interface toolkit" category = "main" optional = false @@ -107,7 +103,7 @@ importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} [[package]] name = "colorama" -version = "0.4.4" +version = "0.4.5" description = "Cross-platform colored terminal text." category = "main" optional = false @@ -142,7 +138,7 @@ python-versions = ">=3.5" [[package]] name = "distlib" -version = "0.3.4" +version = "0.3.5" description = "Distribution utilities" category = "dev" optional = false @@ -150,15 +146,15 @@ python-versions = "*" [[package]] name = "filelock" -version = "3.6.0" +version = "3.8.0" description = "A platform independent file lock." category = "dev" optional = false python-versions = ">=3.7" [package.extras] -docs = ["furo (>=2021.8.17b43)", "sphinx (>=4.1)", "sphinx-autodoc-typehints (>=1.12)"] -testing = ["covdefaults (>=1.2.0)", "coverage (>=4)", "pytest (>=4)", "pytest-cov", "pytest-timeout (>=1.4.2)"] +docs = ["furo (>=2022.6.21)", "sphinx (>=5.1.1)", "sphinx-autodoc-typehints (>=1.19.1)"] +testing = ["covdefaults (>=2.2)", "coverage (>=6.4.2)", "pytest (>=7.1.2)", "pytest-cov (>=3)", "pytest-timeout (>=2.1)"] [[package]] name = "flake8" @@ -195,7 +191,7 @@ python-versions = ">=3.5" [[package]] name = "importlib-metadata" -version = "4.11.3" +version = "4.12.0" description = "Read metadata from Python packages" category = "main" optional = false @@ -208,7 +204,7 @@ zipp = ">=0.5" [package.extras] docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] perf = ["ipython"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] [[package]] name = "inflection" @@ -373,15 +369,15 @@ dev = ["pylint", "ipython", "ipdb", "nose"] [[package]] name = "platformdirs" -version = "2.5.1" +version = "2.5.2" description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." category = "dev" optional = false python-versions = ">=3.7" [package.extras] -docs = ["Sphinx (>=4)", "furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)"] -test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)"] +docs = ["furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)", "sphinx (>=4)"] +test = ["appdirs (==1.4.4)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)", "pytest (>=6)"] [[package]] name = "pluggy" @@ -454,20 +450,21 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pyjwt" -version = "1.7.1" +version = "2.4.0" description = "JSON Web Token implementation in Python" category = "main" optional = false -python-versions = "*" +python-versions = ">=3.6" [package.extras] -crypto = ["cryptography (>=1.4)"] -flake8 = ["flake8", "flake8-import-order", "pep8-naming"] -test = ["pytest (>=4.0.1,<5.0.0)", "pytest-cov (>=2.6.0,<3.0.0)", "pytest-runner (>=4.2,<5.0.0)"] +crypto = ["cryptography (>=3.3.1)"] +dev = ["sphinx", "sphinx-rtd-theme", "zope.interface", "cryptography (>=3.3.1)", "pytest (>=6.0.0,<7.0.0)", "coverage[toml] (==5.0.4)", "mypy", "pre-commit"] +docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] +tests = ["pytest (>=6.0.0,<7.0.0)", "coverage[toml] (==5.0.4)"] [[package]] name = "pyparsing" -version = "3.0.8" +version = "3.0.9" description = "pyparsing module - Classes and methods to define and execute parsing grammars" category = "dev" optional = false @@ -535,21 +532,37 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "requests" -version = "2.27.1" +version = "2.28.1" description = "Python HTTP for Humans." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +python-versions = ">=3.7, <4" [package.dependencies] certifi = ">=2017.4.17" -charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""} -idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""} +charset-normalizer = ">=2,<3" +idna = ">=2.5,<4" urllib3 = ">=1.21.1,<1.27" [package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] -use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-mock" +version = "1.9.3" +description = "Mock out responses from the requests package" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "sphinx", "testrepository (>=0.0.18)", "testtools"] [[package]] name = "simplejson" @@ -561,7 +574,7 @@ python-versions = "*" [[package]] name = "singer-sdk" -version = "0.4.5" +version = "0.4.9" description = "A framework for building Singer taps" category = "main" optional = false @@ -578,12 +591,12 @@ jsonpath-ng = ">=1.5.3,<2.0.0" memoization = ">=0.3.2,<0.4.0" pendulum = ">=2.1.0,<3.0.0" pipelinewise-singer-python = "1.2.0" -PyJWT = "1.7.1" +PyJWT = ">=2.3,<3.0" requests = ">=2.25.1,<3.0.0" sqlalchemy = ">=1.4,<2.0" [package.extras] -docs = ["sphinx (>=3.5.4,<4.0.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinx-copybutton (>=0.3.1,<0.4.0)", "myst-parser (>=0.14.0,<0.15.0)"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinx-copybutton (>=0.3.1,<0.4.0)", "myst-parser (>=0.17.2,<0.18.0)"] [[package]] name = "six" @@ -603,7 +616,7 @@ python-versions = "*" [[package]] name = "sqlalchemy" -version = "1.4.35" +version = "1.4.40" description = "Database Abstraction Library" category = "main" optional = false @@ -618,7 +631,7 @@ aiomysql = ["greenlet (!=0.4.17)", "aiomysql"] aiosqlite = ["typing_extensions (!=3.10.0.1)", "greenlet (!=0.4.17)", "aiosqlite"] asyncio = ["greenlet (!=0.4.17)"] asyncmy = ["greenlet (!=0.4.17)", "asyncmy (>=0.2.3,!=0.2.4)"] -mariadb_connector = ["mariadb (>=1.0.1)"] +mariadb_connector = ["mariadb (>=1.0.1,!=1.1.2)"] mssql = ["pyodbc"] mssql_pymssql = ["pymssql"] mssql_pyodbc = ["pyodbc"] @@ -628,7 +641,7 @@ mysql_connector = ["mysql-connector-python"] oracle = ["cx_oracle (>=7,<8)", "cx_oracle (>=7)"] postgresql = ["psycopg2 (>=2.7)"] postgresql_asyncpg = ["greenlet (!=0.4.17)", "asyncpg"] -postgresql_pg8000 = ["pg8000 (>=1.16.6)"] +postgresql_pg8000 = ["pg8000 (>=1.16.6,!=1.29.0)"] postgresql_psycopg2binary = ["psycopg2-binary"] postgresql_psycopg2cffi = ["psycopg2cffi"] pymysql = ["pymysql (<1)", "pymysql"] @@ -644,15 +657,15 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "tomli" -version = "1.2.3" +version = "2.0.1" description = "A lil' TOML parser" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [[package]] name = "tox" -version = "3.25.0" +version = "3.25.1" description = "tox is a generic virtualenv management and test command line tool" category = "dev" optional = false @@ -683,7 +696,7 @@ python-versions = "*" [[package]] name = "types-requests" -version = "2.27.16" +version = "2.28.8" description = "Typing stubs for requests" category = "dev" optional = false @@ -694,7 +707,7 @@ types-urllib3 = "<1.27" [[package]] name = "types-urllib3" -version = "1.26.11" +version = "1.26.22" description = "Typing stubs for urllib3" category = "dev" optional = false @@ -702,19 +715,19 @@ python-versions = "*" [[package]] name = "typing-extensions" -version = "4.1.1" -description = "Backported and Experimental Type Hints for Python 3.6+" +version = "4.3.0" +description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [[package]] name = "urllib3" -version = "1.26.9" +version = "1.26.11" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" [package.extras] brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] @@ -723,45 +736,41 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.14.1" +version = "20.16.3" description = "Virtual Python Environment builder" category = "dev" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +python-versions = ">=3.6" [package.dependencies] -distlib = ">=0.3.1,<1" -filelock = ">=3.2,<4" -importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} -platformdirs = ">=2,<3" -six = ">=1.9.0,<2" +distlib = ">=0.3.5,<1" +filelock = ">=3.4.1,<4" +importlib-metadata = {version = ">=4.8.3", markers = "python_version < \"3.8\""} +platformdirs = ">=2.4,<3" [package.extras] -docs = ["proselint (>=0.10.2)", "sphinx (>=3)", "sphinx-argparse (>=0.2.5)", "sphinx-rtd-theme (>=0.4.3)", "towncrier (>=21.3)"] -testing = ["coverage (>=4)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", "pytest (>=4)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.1)", "pytest-mock (>=2)", "pytest-randomly (>=1)", "pytest-timeout (>=1)", "packaging (>=20.0)"] +docs = ["proselint (>=0.13)", "sphinx (>=5.1.1)", "sphinx-argparse (>=0.3.1)", "sphinx-rtd-theme (>=1)", "towncrier (>=21.9)"] +testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] [[package]] name = "zipp" -version = "3.8.0" +version = "3.8.1" description = "Backport of pathlib-compatible object wrapper for zip files" category = "main" optional = false python-versions = ">=3.7" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] +docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] [metadata] lock-version = "1.1" python-versions = "<3.11,>=3.7.1" -content-hash = "47b38b0f8633f06fe89e6443d08a0f71823e7e6771662bef059849d3bbf09935" +content-hash = "ca6720cdd7023aa23fa9aa4b5bc09b36866dd85a633abf21f533e55512db9af8" [metadata.files] -atomicwrites = [ - {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, - {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, -] +atomicwrites = [] attrs = [ {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, @@ -770,81 +779,15 @@ backoff = [ {file = "backoff-1.8.0-py2.py3-none-any.whl", hash = "sha256:d340bb6f36d025c04214b8925112d8456970e5f28dda46e4f1133bf5c622cb0a"}, {file = "backoff-1.8.0.tar.gz", hash = "sha256:c7187f15339e775aec926dc6e5e42f8a3ad7d3c2b9a6ecae7b535000f70cd838"}, ] -black = [ - {file = "black-21.12b0-py3-none-any.whl", hash = "sha256:a615e69ae185e08fdd73e4715e260e2479c861b5740057fde6e8b4e3b7dd589f"}, - {file = "black-21.12b0.tar.gz", hash = "sha256:77b80f693a569e2e527958459634f18df9b0ba2625ba4e0c2d5da5be42e6f2b3"}, -] -certifi = [ - {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"}, - {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"}, -] -cffi = [ - {file = "cffi-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962"}, - {file = "cffi-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0"}, - {file = "cffi-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:41d45de54cd277a7878919867c0f08b0cf817605e4eb94093e7516505d3c8d14"}, - {file = "cffi-1.15.0-cp27-cp27m-win32.whl", hash = "sha256:4a306fa632e8f0928956a41fa8e1d6243c71e7eb59ffbd165fc0b41e316b2474"}, - {file = "cffi-1.15.0-cp27-cp27m-win_amd64.whl", hash = "sha256:e7022a66d9b55e93e1a845d8c9eba2a1bebd4966cd8bfc25d9cd07d515b33fa6"}, - {file = "cffi-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:14cd121ea63ecdae71efa69c15c5543a4b5fbcd0bbe2aad864baca0063cecf27"}, - {file = "cffi-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:d4d692a89c5cf08a8557fdeb329b82e7bf609aadfaed6c0d79f5a449a3c7c023"}, - {file = "cffi-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0104fb5ae2391d46a4cb082abdd5c69ea4eab79d8d44eaaf79f1b1fd806ee4c2"}, - {file = "cffi-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:91ec59c33514b7c7559a6acda53bbfe1b283949c34fe7440bcf917f96ac0723e"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f5c7150ad32ba43a07c4479f40241756145a1f03b43480e058cfd862bf5041c7"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abb9a20a72ac4e0fdb50dae135ba5e77880518e742077ced47eb1499e29a443c"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a5263e363c27b653a90078143adb3d076c1a748ec9ecc78ea2fb916f9b861962"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f54a64f8b0c8ff0b64d18aa76675262e1700f3995182267998c31ae974fbc382"}, - {file = "cffi-1.15.0-cp310-cp310-win32.whl", hash = "sha256:c21c9e3896c23007803a875460fb786118f0cdd4434359577ea25eb556e34c55"}, - {file = "cffi-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:5e069f72d497312b24fcc02073d70cb989045d1c91cbd53979366077959933e0"}, - {file = "cffi-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:64d4ec9f448dfe041705426000cc13e34e6e5bb13736e9fd62e34a0b0c41566e"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2756c88cbb94231c7a147402476be2c4df2f6078099a6f4a480d239a8817ae39"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b96a311ac60a3f6be21d2572e46ce67f09abcf4d09344c49274eb9e0bf345fc"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75e4024375654472cc27e91cbe9eaa08567f7fbdf822638be2814ce059f58032"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:59888172256cac5629e60e72e86598027aca6bf01fa2465bdb676d37636573e8"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:27c219baf94952ae9d50ec19651a687b826792055353d07648a5695413e0c605"}, - {file = "cffi-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:4958391dbd6249d7ad855b9ca88fae690783a6be9e86df65865058ed81fc860e"}, - {file = "cffi-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:f6f824dc3bce0edab5f427efcfb1d63ee75b6fcb7282900ccaf925be84efb0fc"}, - {file = "cffi-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:06c48159c1abed75c2e721b1715c379fa3200c7784271b3c46df01383b593636"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c2051981a968d7de9dd2d7b87bcb9c939c74a34626a6e2f8181455dd49ed69e4"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:91d77d2a782be4274da750752bb1650a97bfd8f291022b379bb8e01c66b4e96b"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:45db3a33139e9c8f7c09234b5784a5e33d31fd6907800b316decad50af323ff2"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:263cc3d821c4ab2213cbe8cd8b355a7f72a8324577dc865ef98487c1aeee2bc7"}, - {file = "cffi-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:17771976e82e9f94976180f76468546834d22a7cc404b17c22df2a2c81db0c66"}, - {file = "cffi-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:3415c89f9204ee60cd09b235810be700e993e343a408693e80ce7f6a40108029"}, - {file = "cffi-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4238e6dab5d6a8ba812de994bbb0a79bddbdf80994e4ce802b6f6f3142fcc880"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0808014eb713677ec1292301ea4c81ad277b6cdf2fdd90fd540af98c0b101d20"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:57e9ac9ccc3101fac9d6014fba037473e4358ef4e89f8e181f8951a2c0162024"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b6c2ea03845c9f501ed1313e78de148cd3f6cad741a75d43a29b43da27f2e1e"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10dffb601ccfb65262a27233ac273d552ddc4d8ae1bf93b21c94b8511bffe728"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:786902fb9ba7433aae840e0ed609f45c7bcd4e225ebb9c753aa39725bb3e6ad6"}, - {file = "cffi-1.15.0-cp38-cp38-win32.whl", hash = "sha256:da5db4e883f1ce37f55c667e5c0de439df76ac4cb55964655906306918e7363c"}, - {file = "cffi-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:181dee03b1170ff1969489acf1c26533710231c58f95534e3edac87fff06c443"}, - {file = "cffi-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:45e8636704eacc432a206ac7345a5d3d2c62d95a507ec70d62f23cd91770482a"}, - {file = "cffi-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:31fb708d9d7c3f49a60f04cf5b119aeefe5644daba1cd2a0fe389b674fd1de37"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6dc2737a3674b3e344847c8686cf29e500584ccad76204efea14f451d4cc669a"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:74fdfdbfdc48d3f47148976f49fab3251e550a8720bebc99bf1483f5bfb5db3e"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f7d084648d77af029acb79a0ff49a0ad7e9d09057a9bf46596dac9514dc07df"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef1f279350da2c586a69d32fc8733092fd32cc8ac95139a00377841f59a3f8d8"}, - {file = "cffi-1.15.0-cp39-cp39-win32.whl", hash = "sha256:2a23af14f408d53d5e6cd4e3d9a24ff9e05906ad574822a10563efcef137979a"}, - {file = "cffi-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139"}, - {file = "cffi-1.15.0.tar.gz", hash = "sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954"}, -] -charset-normalizer = [ - {file = "charset-normalizer-2.0.12.tar.gz", hash = "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597"}, - {file = "charset_normalizer-2.0.12-py3-none-any.whl", hash = "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"}, -] +black = [] +certifi = [] +cffi = [] +charset-normalizer = [] ciso8601 = [ {file = "ciso8601-2.2.0.tar.gz", hash = "sha256:14ad817ed31a698372d42afa81b0173d71cd1d0b48b7499a2da2a01dcc8695e6"}, ] -click = [ - {file = "click-8.1.2-py3-none-any.whl", hash = "sha256:24e1a4a9ec5bf6299411369b208c1df2188d9eb8d916302fe6bf03faed227f1e"}, - {file = "click-8.1.2.tar.gz", hash = "sha256:479707fe14d9ec9a0757618b7a100a0ae4c4e236fac5b7f80ca68028141a1a72"}, -] -colorama = [ - {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, - {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, -] +click = [] +colorama = [] cryptography = [ {file = "cryptography-3.4.8-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:a00cf305f07b26c351d8d4e1af84ad7501eca8a342dedf24a7acb0e7b7406e14"}, {file = "cryptography-3.4.8-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:f44d141b8c4ea5eb4dbc9b3ad992d45580c1d22bf5e24363f2fbf50c2d7ae8a7"}, @@ -870,14 +813,8 @@ decorator = [ {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"}, {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"}, ] -distlib = [ - {file = "distlib-0.3.4-py2.py3-none-any.whl", hash = "sha256:6564fe0a8f51e734df6333d08b8b94d4ea8ee6b99b5ed50613f731fd4089f34b"}, - {file = "distlib-0.3.4.zip", hash = "sha256:e4b58818180336dc9c529bfb9a0b58728ffc09ad92027a3f30b7cd91e3458579"}, -] -filelock = [ - {file = "filelock-3.6.0-py3-none-any.whl", hash = "sha256:f8314284bfffbdcfa0ff3d7992b023d4c628ced6feb957351d4c48d059f56bc0"}, - {file = "filelock-3.6.0.tar.gz", hash = "sha256:9cd540a9352e432c7246a48fe4e8712b10acb1df2ad1f30e8c070b82ae1fed85"}, -] +distlib = [] +filelock = [] flake8 = [ {file = "flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"}, {file = "flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"}, @@ -894,7 +831,6 @@ greenlet = [ {file = "greenlet-1.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:97e5306482182170ade15c4b0d8386ded995a07d7cc2ca8f27958d34d6736497"}, {file = "greenlet-1.1.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e6a36bb9474218c7a5b27ae476035497a6990e21d04c279884eb10d9b290f1b1"}, {file = "greenlet-1.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abb7a75ed8b968f3061327c433a0fbd17b729947b400747c334a9c29a9af6c58"}, - {file = "greenlet-1.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b336501a05e13b616ef81ce329c0e09ac5ed8c732d9ba7e3e983fcc1a9e86965"}, {file = "greenlet-1.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:14d4f3cd4e8b524ae9b8aa567858beed70c392fdec26dbdb0a8a418392e71708"}, {file = "greenlet-1.1.2-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:17ff94e7a83aa8671a25bf5b59326ec26da379ace2ebc4411d690d80a7fbcf23"}, {file = "greenlet-1.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9f3cba480d3deb69f6ee2c1825060177a22c7826431458c697df88e6aeb3caee"}, @@ -907,7 +843,6 @@ greenlet = [ {file = "greenlet-1.1.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9d29ca8a77117315101425ec7ec2a47a22ccf59f5593378fc4077ac5b754fce"}, {file = "greenlet-1.1.2-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:21915eb821a6b3d9d8eefdaf57d6c345b970ad722f856cd71739493ce003ad08"}, {file = "greenlet-1.1.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eff9d20417ff9dcb0d25e2defc2574d10b491bf2e693b4e491914738b7908168"}, - {file = "greenlet-1.1.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b8c008de9d0daba7b6666aa5bbfdc23dcd78cafc33997c9b7741ff6353bafb7f"}, {file = "greenlet-1.1.2-cp36-cp36m-win32.whl", hash = "sha256:32ca72bbc673adbcfecb935bb3fb1b74e663d10a4b241aaa2f5a75fe1d1f90aa"}, {file = "greenlet-1.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f0214eb2a23b85528310dad848ad2ac58e735612929c8072f6093f3585fd342d"}, {file = "greenlet-1.1.2-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:b92e29e58bef6d9cfd340c72b04d74c4b4e9f70c9fa7c78b674d1fec18896dc4"}, @@ -916,7 +851,6 @@ greenlet = [ {file = "greenlet-1.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e12bdc622676ce47ae9abbf455c189e442afdde8818d9da983085df6312e7a1"}, {file = "greenlet-1.1.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8c790abda465726cfb8bb08bd4ca9a5d0a7bd77c7ac1ca1b839ad823b948ea28"}, {file = "greenlet-1.1.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f276df9830dba7a333544bd41070e8175762a7ac20350786b322b714b0e654f5"}, - {file = "greenlet-1.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c5d5b35f789a030ebb95bff352f1d27a93d81069f2adb3182d99882e095cefe"}, {file = "greenlet-1.1.2-cp37-cp37m-win32.whl", hash = "sha256:64e6175c2e53195278d7388c454e0b30997573f3f4bd63697f88d855f7a6a1fc"}, {file = "greenlet-1.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b11548073a2213d950c3f671aa88e6f83cda6e2fb97a8b6317b1b5b33d850e06"}, {file = "greenlet-1.1.2-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:9633b3034d3d901f0a46b7939f8c4d64427dfba6bbc5a36b1a67364cf148a1b0"}, @@ -925,7 +859,6 @@ greenlet = [ {file = "greenlet-1.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e859fcb4cbe93504ea18008d1df98dee4f7766db66c435e4882ab35cf70cac43"}, {file = "greenlet-1.1.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:00e44c8afdbe5467e4f7b5851be223be68adb4272f44696ee71fe46b7036a711"}, {file = "greenlet-1.1.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec8c433b3ab0419100bd45b47c9c8551248a5aee30ca5e9d399a0b57ac04651b"}, - {file = "greenlet-1.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2bde6792f313f4e918caabc46532aa64aa27a0db05d75b20edfc5c6f46479de2"}, {file = "greenlet-1.1.2-cp38-cp38-win32.whl", hash = "sha256:288c6a76705dc54fba69fbcb59904ae4ad768b4c768839b8ca5fdadec6dd8cfd"}, {file = "greenlet-1.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:8d2f1fb53a421b410751887eb4ff21386d119ef9cde3797bf5e7ed49fb51a3b3"}, {file = "greenlet-1.1.2-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:166eac03e48784a6a6e0e5f041cfebb1ab400b394db188c48b3a84737f505b67"}, @@ -934,7 +867,6 @@ greenlet = [ {file = "greenlet-1.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1692f7d6bc45e3200844be0dba153612103db241691088626a33ff1f24a0d88"}, {file = "greenlet-1.1.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7227b47e73dedaa513cdebb98469705ef0d66eb5a1250144468e9c3097d6b59b"}, {file = "greenlet-1.1.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ff61ff178250f9bb3cd89752df0f1dd0e27316a8bd1465351652b1b4a4cdfd3"}, - {file = "greenlet-1.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0051c6f1f27cb756ffc0ffbac7d2cd48cb0362ac1736871399a739b2885134d3"}, {file = "greenlet-1.1.2-cp39-cp39-win32.whl", hash = "sha256:f70a9e237bb792c7cc7e44c531fd48f5897961701cdaa06cf22fc14965c496cf"}, {file = "greenlet-1.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:013d61294b6cd8fe3242932c1c5e36e5d1db2c8afb58606c5a67efce62c1f5fd"}, {file = "greenlet-1.1.2.tar.gz", hash = "sha256:e30f5ea4ae2346e62cedde8794a56858a67b878dd79f7df76a0767e356b1744a"}, @@ -943,10 +875,7 @@ idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, ] -importlib-metadata = [ - {file = "importlib_metadata-4.11.3-py3-none-any.whl", hash = "sha256:1208431ca90a8cca1a6b8af391bb53c1a2db74e5d1cef6ddced95d4b2062edc6"}, - {file = "importlib_metadata-4.11.3.tar.gz", hash = "sha256:ea4c597ebf37142f827b8f39299579e31685c31d3a438b59f469406afd0f2539"}, -] +importlib-metadata = [] inflection = [ {file = "inflection-0.5.1-py2.py3-none-any.whl", hash = "sha256:f38b2b640938a4f35ade69ac3d053042959b62a0f1076a5bbaa1b9526605a8a2"}, {file = "inflection-0.5.1.tar.gz", hash = "sha256:1a29730d366e996aaacffb2f1f1cb9593dc38e2ddd30c91250c6dde09ea9b417"}, @@ -1044,10 +973,7 @@ pipelinewise-singer-python = [ {file = "pipelinewise-singer-python-1.2.0.tar.gz", hash = "sha256:8ba501f9092dbd686cd5792ecf6aa97c2d25c225e9d8b2875dcead0f5738898c"}, {file = "pipelinewise_singer_python-1.2.0-py3-none-any.whl", hash = "sha256:156f011cba10b1591ae37c5510ed9d21639258c1377cc00c07d9f7e9a3ae27fb"}, ] -platformdirs = [ - {file = "platformdirs-2.5.1-py3-none-any.whl", hash = "sha256:bcae7cab893c2d310a711b70b24efb93334febe65f8de776ee320b517471e227"}, - {file = "platformdirs-2.5.1.tar.gz", hash = "sha256:7535e70dfa32e84d4b34996ea99c5e432fa29a708d0f4e394bbcb2a8faa4f16d"}, -] +platformdirs = [] pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, @@ -1076,14 +1002,8 @@ pyflakes = [ {file = "pyflakes-2.3.1-py2.py3-none-any.whl", hash = "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3"}, {file = "pyflakes-2.3.1.tar.gz", hash = "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"}, ] -pyjwt = [ - {file = "PyJWT-1.7.1-py2.py3-none-any.whl", hash = "sha256:5c6eca3c2940464d106b99ba83b00c6add741c9becaec087fb7ccdefea71350e"}, - {file = "PyJWT-1.7.1.tar.gz", hash = "sha256:8d59a976fb773f3e6a39c85636357c4f0e242707394cadadd9814f5cbaa20e96"}, -] -pyparsing = [ - {file = "pyparsing-3.0.8-py3-none-any.whl", hash = "sha256:ef7b523f6356f763771559412c0d7134753f037822dad1b16945b7b846f7ad06"}, - {file = "pyparsing-3.0.8.tar.gz", hash = "sha256:7bf433498c016c4314268d95df76c81b842a4cb2b276fa3312cfb1e1d85f6954"}, -] +pyjwt = [] +pyparsing = [] pyrsistent = [ {file = "pyrsistent-0.18.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:df46c854f490f81210870e509818b729db4488e1f30f2a1ce1698b2295a878d1"}, {file = "pyrsistent-0.18.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d45866ececf4a5fff8742c25722da6d4c9e180daa7b405dc0a2a2790d668c26"}, @@ -1123,9 +1043,10 @@ pytzdata = [ {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, ] -requests = [ - {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"}, - {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"}, +requests = [] +requests-mock = [ + {file = "requests-mock-1.9.3.tar.gz", hash = "sha256:8d72abe54546c1fc9696fa1516672f1031d72a55a1d66c85184f972a24ba0eba"}, + {file = "requests_mock-1.9.3-py2.py3-none-any.whl", hash = "sha256:0a2d38a117c08bb78939ec163522976ad59a6b7fdd82b709e23bb98004a44970"}, ] simplejson = [ {file = "simplejson-3.11.1-cp27-cp27m-win32.whl", hash = "sha256:38c2b563cd03363e7cb2bbba6c20ae4eaafd853a83954c8c8dd345ee391787bf"}, @@ -1146,10 +1067,7 @@ simplejson = [ {file = "simplejson-3.11.1.win32-py3.4.exe", hash = "sha256:97cc43ef4cb18a2725f6e26d22b96f8ca50872a195bde32707dcb284f89c1d4d"}, {file = "simplejson-3.11.1.win32-py3.5.exe", hash = "sha256:c76d55d78dc8b06c96fd08c6cc5e2b0b650799627d3f9ca4ad23f40db72d5f6d"}, ] -singer-sdk = [ - {file = "singer-sdk-0.4.5.tar.gz", hash = "sha256:120f7857bf238b5e95aeeba459c0c7178da5722d5d8f43899a40fd06a23fe22b"}, - {file = "singer_sdk-0.4.5-py3-none-any.whl", hash = "sha256:ff0d05f39a5ddef38d8db8b7b8aee3f86623b252aa4182a8e05279a6496b4a50"}, -] +singer-sdk = [] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, @@ -1158,56 +1076,16 @@ snowballstemmer = [ {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"}, {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, ] -sqlalchemy = [ - {file = "SQLAlchemy-1.4.35-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:093b3109c2747d5dc0fa4314b1caf4c7ca336d5c8c831e3cfbec06a7e861e1e6"}, - {file = "SQLAlchemy-1.4.35-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c6fb6b9ed1d0be7fa2c90be8ad2442c14cbf84eb0709dd1afeeff1e511550041"}, - {file = "SQLAlchemy-1.4.35-cp27-cp27m-win32.whl", hash = "sha256:d38a49aa75a5759d0d118e26701d70c70a37b896379115f8386e91b0444bfa70"}, - {file = "SQLAlchemy-1.4.35-cp27-cp27m-win_amd64.whl", hash = "sha256:70e571ae9ee0ff36ed37e2b2765445d54981e4d600eccdf6fe3838bc2538d157"}, - {file = "SQLAlchemy-1.4.35-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:48036698f20080462e981b18d77d574631a3d1fc2c33b416c6df299ec1d10b99"}, - {file = "SQLAlchemy-1.4.35-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:4ba2c1f368bcf8551cdaa27eac525022471015633d5bdafbc4297e0511f62f51"}, - {file = "SQLAlchemy-1.4.35-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d17316100fcd0b6371ac9211351cb976fd0c2e12a859c1a57965e3ef7f3ed2bc"}, - {file = "SQLAlchemy-1.4.35-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9837133b89ad017e50a02a3b46419869cf4e9aa02743e911b2a9e25fa6b05403"}, - {file = "SQLAlchemy-1.4.35-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a4efb70a62cbbbc052c67dc66b5448b0053b509732184af3e7859d05fdf6223c"}, - {file = "SQLAlchemy-1.4.35-cp310-cp310-win32.whl", hash = "sha256:1ff9f84b2098ef1b96255a80981ee10f4b5d49b6cfeeccf9632c2078cd86052e"}, - {file = "SQLAlchemy-1.4.35-cp310-cp310-win_amd64.whl", hash = "sha256:48f0eb5bcc87a9b2a95b345ed18d6400daaa86ca414f6840961ed85c342af8f4"}, - {file = "SQLAlchemy-1.4.35-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:da25e75ba9f3fabc271673b6b413ca234994e6d3453424bea36bb5549c5bbaec"}, - {file = "SQLAlchemy-1.4.35-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aeea6ace30603ca9a8869853bb4a04c7446856d7789e36694cd887967b7621f6"}, - {file = "SQLAlchemy-1.4.35-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a5dbdbb39c1b100df4d182c78949158073ca46ba2850c64fe02ffb1eb5b70903"}, - {file = "SQLAlchemy-1.4.35-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cfd8e4c64c30a5219032e64404d468c425bdbc13b397da906fc9bee6591fc0dd"}, - {file = "SQLAlchemy-1.4.35-cp36-cp36m-win32.whl", hash = "sha256:9dac1924611698f8fe5b2e58601156c01da2b6c0758ba519003013a78280cf4d"}, - {file = "SQLAlchemy-1.4.35-cp36-cp36m-win_amd64.whl", hash = "sha256:e8b09e2d90267717d850f2e2323919ea32004f55c40e5d53b41267e382446044"}, - {file = "SQLAlchemy-1.4.35-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:63c82c9e8ccc2fb4bfd87c24ffbac320f70b7c93b78f206c1f9c441fa3013a5f"}, - {file = "SQLAlchemy-1.4.35-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:effadcda9a129cc56408dd5b2ea20ee9edcea24bd58e6a1489fa27672d733182"}, - {file = "SQLAlchemy-1.4.35-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2c6c411d8c59afba95abccd2b418f30ade674186660a2d310d364843049fb2c1"}, - {file = "SQLAlchemy-1.4.35-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2489e70bfa2356f2d421106794507daccf6cc8711753c442fc97272437fc606"}, - {file = "SQLAlchemy-1.4.35-cp37-cp37m-win32.whl", hash = "sha256:186cb3bd77abf2ddcf722f755659559bfb157647b3fd3f32ea1c70e8311e8f6b"}, - {file = "SQLAlchemy-1.4.35-cp37-cp37m-win_amd64.whl", hash = "sha256:babd63fb7cb6b0440abb6d16aca2be63342a6eea3dc7b613bb7a9357dc36920f"}, - {file = "SQLAlchemy-1.4.35-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:9e1a72197529ea00357640f21d92ffc7024e156ef9ac36edf271c8335facbc1a"}, - {file = "SQLAlchemy-1.4.35-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e255a8dd5572b0c66d6ee53597d36157ad6cf3bc1114f61c54a65189f996ab03"}, - {file = "SQLAlchemy-1.4.35-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9bec63b1e20ef69484f530fb4b4837e050450637ff9acd6dccc7003c5013abf8"}, - {file = "SQLAlchemy-1.4.35-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95411abc0e36d18f54fa5e24d42960ea3f144fb16caaa5a8c2e492b5424cc82c"}, - {file = "SQLAlchemy-1.4.35-cp38-cp38-win32.whl", hash = "sha256:28b17ebbaee6587013be2f78dc4f6e95115e1ec8dd7647c4e7be048da749e48b"}, - {file = "SQLAlchemy-1.4.35-cp38-cp38-win_amd64.whl", hash = "sha256:9e7094cf04e6042c4210a185fa7b9b8b3b789dd6d1de7b4f19452290838e48bd"}, - {file = "SQLAlchemy-1.4.35-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:1b4eac3933c335d7f375639885765722534bb4e52e51cdc01a667eea822af9b6"}, - {file = "SQLAlchemy-1.4.35-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d8edfb09ed2b865485530c13e269833dab62ab2d582fde21026c9039d4d0e62"}, - {file = "SQLAlchemy-1.4.35-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6204d06bfa85f87625e1831ca663f9dba91ac8aec24b8c65d02fb25cbaf4b4d7"}, - {file = "SQLAlchemy-1.4.35-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28aa2ef06c904729620cc735262192e622db9136c26d8587f71f29ec7715628a"}, - {file = "SQLAlchemy-1.4.35-cp39-cp39-win32.whl", hash = "sha256:ecc81336b46e31ae9c9bdfa220082079914e31a476d088d3337ecf531d861228"}, - {file = "SQLAlchemy-1.4.35-cp39-cp39-win_amd64.whl", hash = "sha256:53c7469b86a60fe2babca4f70111357e6e3d5150373bc85eb3b914356983e89a"}, - {file = "SQLAlchemy-1.4.35.tar.gz", hash = "sha256:2ffc813b01dc6473990f5e575f210ca5ac2f5465ace3908b78ffd6d20058aab5"}, -] +sqlalchemy = [] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] tomli = [ - {file = "tomli-1.2.3-py3-none-any.whl", hash = "sha256:e3069e4be3ead9668e21cb9b074cd948f7b3113fd9c8bba083f48247aab8b11c"}, - {file = "tomli-1.2.3.tar.gz", hash = "sha256:05b6166bff487dc068d322585c7ea4ef78deed501cc124060e0f238e89a9231f"}, -] -tox = [ - {file = "tox-3.25.0-py2.py3-none-any.whl", hash = "sha256:0805727eb4d6b049de304977dfc9ce315a1938e6619c3ab9f38682bb04662a5a"}, - {file = "tox-3.25.0.tar.gz", hash = "sha256:37888f3092aa4e9f835fc8cc6dadbaaa0782651c41ef359e3a5743fcb0308160"}, + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +tox = [] typed-ast = [ {file = "typed_ast-1.4.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:2068531575a125b87a41802130fa7e29f26c09a2833fea68d9a40cf33902eba6"}, {file = "typed_ast-1.4.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:c907f561b1e83e93fad565bac5ba9c22d96a54e7ea0267c708bffe863cbe4075"}, @@ -1240,27 +1118,9 @@ typed-ast = [ {file = "typed_ast-1.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:9c6d1a54552b5330bc657b7ef0eae25d00ba7ffe85d9ea8ae6540d2197a3788c"}, {file = "typed_ast-1.4.3.tar.gz", hash = "sha256:fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65"}, ] -types-requests = [ - {file = "types-requests-2.27.16.tar.gz", hash = "sha256:c8010c18b291a7efb60b1452dbe12530bc25693dd657e70c62803fcdc4bffe9b"}, - {file = "types_requests-2.27.16-py3-none-any.whl", hash = "sha256:2437a5f4d16c0c8bd7539a8126d492b7aeb41e6cda670d76b286c7f83a658d42"}, -] -types-urllib3 = [ - {file = "types-urllib3-1.26.11.tar.gz", hash = "sha256:24d64e441168851eb05f1d022de18ae31558f5649c8f1117e384c2e85e31315b"}, - {file = "types_urllib3-1.26.11-py3-none-any.whl", hash = "sha256:bd0abc01e9fb963e4fddd561a56d21cc371b988d1245662195c90379077139cd"}, -] -typing-extensions = [ - {file = "typing_extensions-4.1.1-py3-none-any.whl", hash = "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2"}, - {file = "typing_extensions-4.1.1.tar.gz", hash = "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42"}, -] -urllib3 = [ - {file = "urllib3-1.26.9-py2.py3-none-any.whl", hash = "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14"}, - {file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"}, -] -virtualenv = [ - {file = "virtualenv-20.14.1-py2.py3-none-any.whl", hash = "sha256:e617f16e25b42eb4f6e74096b9c9e37713cf10bf30168fb4a739f3fa8f898a3a"}, - {file = "virtualenv-20.14.1.tar.gz", hash = "sha256:ef589a79795589aada0c1c5b319486797c03b67ac3984c48c669c0e4f50df3a5"}, -] -zipp = [ - {file = "zipp-3.8.0-py3-none-any.whl", hash = "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"}, - {file = "zipp-3.8.0.tar.gz", hash = "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad"}, -] +types-requests = [] +types-urllib3 = [] +typing-extensions = [] +urllib3 = [] +virtualenv = [] +zipp = [] diff --git a/pyproject.toml b/pyproject.toml index 283e4bc..d227279 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,11 +20,12 @@ atomicwrites = "^1.4.0" pytest = "^6.2.5" tox = "^3.24.4" flake8 = "^3.9.2" -black = "^21.9b0" +black = "^22.3.0" pydocstyle = "^6.1.1" mypy = "^0.910" types-requests = "^2.26.1" isort = "^5.10.1" +requests-mock = "^1.9.3" [tool.isort] profile = "black" diff --git a/tap_hubspot/client.py b/tap_hubspot/client.py index 21ea81b..35c0fc3 100644 --- a/tap_hubspot/client.py +++ b/tap_hubspot/client.py @@ -1,24 +1,23 @@ """REST client handling, including HubspotStream base class.""" - +import backoff import requests -import json from pathlib import Path -from typing import Any, Dict, Optional, Union, List, Iterable +from typing import Any, Dict, Optional, List, Iterable, Callable import pytz import singer -from memoization import cached - from singer import utils +from singer_sdk.exceptions import RetriableAPIError from singer_sdk.helpers.jsonpath import extract_jsonpath from singer_sdk.streams import RESTStream from singer_sdk.authenticators import BearerTokenAuthenticator -from singer_sdk import typing as th # JSON schema typing helpers +from singer_sdk import typing as th SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") LOGGER = singer.get_logger() + class HubspotStream(RESTStream): """Hubspot stream class.""" @@ -31,12 +30,10 @@ class HubspotStream(RESTStream): cached_schema = None properties = [] - @property def schema_filepath(self) -> Path: return SCHEMAS_DIR / f"{self.name}.json" - @property def authenticator(self) -> BearerTokenAuthenticator: """Return a new authenticator object.""" @@ -159,7 +156,6 @@ def get_custom_schema(self, poorly_cast: List[str] = []): internal_properties: List[th.Property] = [] properties: List[th.Property] = [] - properties_hub = self.get_properties() params = [] @@ -184,10 +180,41 @@ def get_custom_schema(self, poorly_cast: List[str] = []): def get_properties(self) -> List[dict]: response = requests.get(f"{self.url_base}/crm/v3/properties/{self.name}", headers=self.http_headers) res = response.json() - return res['results'] + + try: + return res["results"] + except KeyError: + raise KeyError(f"Error retrieving the API query results: {res}") def get_params_from_properties(self, properties: List[dict]) -> List[str]: params = [] for prop in properties: params.append(prop['name']) return params + + def request_decorator(self, func: Callable) -> Callable: + """Instantiate a decorator for handling request failures. + + Uses a wait generator defined in `backoff_wait_generator` to + determine backoff behaviour. Try limit is defined in + `backoff_max_tries`, and will trigger the event defined in + `backoff_handler` before retrying. Developers may override one or + all of these methods to provide custom backoff or retry handling. + + Args: + func: Function to decorate. + + Returns: + A decorated method. + """ + decorator: Callable = backoff.on_exception( + self.backoff_wait_generator, + ( + RetriableAPIError, + requests.exceptions.ReadTimeout, + requests.exceptions.ConnectionError, + ), + max_tries=self.backoff_max_tries, + on_backoff=self.backoff_handler, + )(func) + return decorator diff --git a/tap_hubspot/marketing_streams.py b/tap_hubspot/marketing_streams.py index 8eae714..732af89 100644 --- a/tap_hubspot/marketing_streams.py +++ b/tap_hubspot/marketing_streams.py @@ -36,7 +36,7 @@ Emails, CampaignIds, Campaigns, - Forms + Forms, ) class MarketingStream(HubspotStream): @@ -154,10 +154,82 @@ def get_url_params( return params - class MarketingFormsStream(MarketingStream): name = "forms_v3" path = "/marketing/v3/forms/" primary_keys = ["id"] schema = Forms.schema + def get_url_params( + self, context: Optional[dict], next_page_token: Optional[Any] + ) -> Dict[str, Any]: + """Return a dictionary of values to be used in URL parameterization.""" + params: dict = {} + if next_page_token: + params["after"] = next_page_token + params["count"] = 100 + params["formTypes"] = "all" + return params + + +class MarketingListsStream(HubspotStream): + """Define stream for Marketing Lists.""" + + # todo: update when Hubspot updates API to v3 + name = "lists_v1" + path = "/contacts/v1/lists" + primary_keys = ["listId"] + replication_method = "FULL_TABLE" + replication_key = "" + next_page_token_jsonpath = "$.offset" + records_jsonpath = "$.lists[*]" + + def get_next_page_token( + self, response: requests.Response, previous_token: Optional[Any] + ) -> Optional[Any]: + """Return a token for identifying next page or None if no more pages.""" + all_matches = extract_jsonpath("$.has-more", response.json()) + has_more = next(iter(all_matches), None) + if has_more: + all_matches = extract_jsonpath( + self.next_page_token_jsonpath + , response.json() + ) + first_offset_match = next(iter(all_matches), None) + return first_offset_match + else: + return None + + def get_url_params(self, context: Optional[dict], next_page_token: Optional[Any]) -> Dict[str, Any]: + params = super().get_url_params(context, next_page_token) + params['count'] = 100 + params['offset'] = next_page_token if next_page_token else 0 + return params + + def get_child_context(self, record: dict, context: Optional[dict]) -> dict: + """Return a context dictionary for child streams.""" + return {"listId": record["listId"]} + + +class MarketingListContactsStream(MarketingListsStream): + records_jsonpath = "$.contacts[*]" + name = "list_contacts_v1" + path = "/contacts/v1/lists/{listId}/contacts/all" + primary_keys = ["canonical-vid", "listId"] + replication_method = "FULL_TABLE" + replication_key = "" + parent_stream_type = MarketingListsStream + next_page_token_jsonpath = "$.vid-offset" + + def get_url_params(self, context: Optional[dict], next_page_token: Optional[Any]) -> Dict[str, Any]: + params = super().get_url_params(context, next_page_token) + params['count'] = 100 + params['vidOffset'] = next_page_token if next_page_token else 0 + return params + + def post_process(self, row: dict, context: Optional[dict]) -> dict: + """As needed, append or transform raw data to match expected structure. + Returns row, or None if row is to be excluded""" + + row["listId"] = context["listId"] + return row diff --git a/tap_hubspot/schemas/contacts.json b/tap_hubspot/schemas/contacts.json index 346ba21..2fb7050 100644 --- a/tap_hubspot/schemas/contacts.json +++ b/tap_hubspot/schemas/contacts.json @@ -14,6 +14,10 @@ }, "archived": { "type": ["null", "boolean"] + }, + "archivedAt": { + "type": ["null", "string"], + "format": "date-time" } } } diff --git a/tap_hubspot/schemas/list_contacts_v1.json b/tap_hubspot/schemas/list_contacts_v1.json new file mode 100644 index 0000000..9b9c2f0 --- /dev/null +++ b/tap_hubspot/schemas/list_contacts_v1.json @@ -0,0 +1,70 @@ +{ + "type": "object", + "properties": { + "listId": { + "type": ["integer", "null"] + }, + "addedAt": { + "type": ["integer", "null"] + }, + "vid": { + "type": ["integer", "null"] + }, + "canonical-vid": { + "type": ["integer", "null"] + }, + "merged-vids": { + "type": ["array", "null", "string"] + }, + "merge-audits": { + "type": ["array", "null", "string"] + }, + "portal-id": { + "type": ["integer", "null"] + }, + "is-contact": { + "type": ["boolean", "null"] + }, + "properties": { + "type": ["object", "null"], + "properties": {} + }, + "form-submissions": { + "type": ["array", "null", "string"] + }, + "identity-profiles": { + "type": ["array", "null", "string"], + "items": { + "type": "object", + "properties": { + "vid": { + "type": ["integer", "null"] + }, + "saved-at-timestamp": { + "type": ["integer", "null"] + }, + "deleted-changed-timestamp": { + "type": ["integer", "null"] + }, + "identities": { + "type": ["array", "null"], + "items": { + "type": "object", + "properties": { + "type": { + "type": ["string", "null"] + }, + "value": { + "type": ["string", "null"] + }, + "timestamp": { + "type": ["integer", "null"] + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/tap_hubspot/schemas/lists_v1.json b/tap_hubspot/schemas/lists_v1.json new file mode 100644 index 0000000..a7395cc --- /dev/null +++ b/tap_hubspot/schemas/lists_v1.json @@ -0,0 +1,84 @@ +{ + "type": "object", + "properties": { + "listId": { + "type": ["integer", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "createAt": { + "type": ["integer", "null"] + }, + "updatedAt": { + "type": ["integer", "null"] + }, + "dynamic": { + "type": ["boolean", "null"] + }, + "filters": { + "type": ["array", "null", "string"], + "items": { + "type": ["array", "null"], + "items": { + "type": ["object", "null"], + "properties": { + "filterFamily": { + "type": ["string", "null"] + }, + "withinTimeMode": { + "type": ["string", "null"] + }, + "checkPastVersions": { + "type": ["boolean", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "property": { + "type": ["string", "null"] + }, + "value": { + "type": ["string", "null"] + }, + "operator": { + "type": ["string", "null"] + } + } + } + } + }, + "metaData": { + "type": ["object", "null"], + "properties": { + "processing": { + "type": ["string", "null"] + }, + "size": { + "type": ["integer", "null"] + }, + "error": { + "type": ["string", "null"] + }, + "lastProcessingStateChangeAt": { + "type": ["integer", "null"] + }, + "lastSizeChangeAt": { + "type": ["integer", "null"] + } + } + }, + "portalId": { + "type": ["integer", "null"] + }, + "listType": { + "type": ["string", "null"] + }, + "internalListId": { + "type": ["integer", "null"] + }, + "deleteable": { + "type": ["boolean", "null"] + } + } +} \ No newline at end of file diff --git a/tap_hubspot/schemas/marketing/Forms.py b/tap_hubspot/schemas/marketing/Forms.py index d59c0f8..0f8d88f 100644 --- a/tap_hubspot/schemas/marketing/Forms.py +++ b/tap_hubspot/schemas/marketing/Forms.py @@ -3,27 +3,29 @@ schema = th.PropertiesList( th.Property("id", th.StringType), th.Property("name", th.StringType), - th.Property("createAt", th.DateTimeType), + th.Property("createdAt", th.DateTimeType), th.Property("updatedAt", th.DateTimeType), th.Property("archived", th.BooleanType), th.Property("fieldGroups", - th.ArrayType( - th.ObjectType( - th.Property("groupType", th.StringType), - th.Property("richTextType", th.StringType), - th.Property("fields", - th.ArrayType( - th.ObjectType( - th.Property("objectTypeId", th.StringType), - th.Property("name", th.StringType), - th.Property("required", th.BooleanType), - th.Property("hidden", th.BooleanType), - th.Property("fieldType", th.StringType), - ) - ) - ) - ) - ) + th.CustomType({"anyOf": [{"type": "string"}, {"type": "null"}, {"type:": "array"}]}) + # todo: doesn't play well with flattening + # th.ArrayType( + # th.ObjectType( + # th.Property("groupType", th.StringType), + # th.Property("richTextType", th.StringType), + # th.Property("fields", + # th.ArrayType( + # th.ObjectType( + # th.Property("objectTypeId", th.StringType), + # th.Property("name", th.StringType), + # th.Property("required", th.BooleanType), + # th.Property("hidden", th.BooleanType), + # th.Property("fieldType", th.StringType), + # ) + # ) + # ) + # ) + # ) ), th.Property("configuration", th.ObjectType( @@ -38,4 +40,41 @@ ) ), th.Property("formType", th.StringType), + th.Property("displayOptions", th.ObjectType( + th.Property("cssClass", th.StringType), + th.Property("renderRawHtml", th.BooleanType), + th.Property("style", th.ObjectType( + th.Property("backgroundWidth", th.StringType), + th.Property("fontFamily", th.StringType), + th.Property("helpTextColor", th.StringType), + th.Property("helpTextSize", th.StringType), + th.Property("labelTextColor", th.StringType), + th.Property("labelTextSize", th.StringType), + th.Property("legalConsentTextColor", th.StringType), + th.Property("legalConsentTextSize", th.StringType), + th.Property("submitAlignment", th.StringType), + th.Property("submitColor", th.StringType), + th.Property("submitFontColor", th.StringType), + th.Property("submitSize", th.StringType), + )), + th.Property("submitButtonText", th.StringType), + th.Property("theme", th.StringType), + )), + th.Property("legalConsentOptions", th.ObjectType( + th.Property("communicationConsentText", th.StringType), + th.Property("communicationsCheckboxes", + th.CustomType({"anyOf": [{"type": "string"}, {"type": "null"}, {"type:": "array"}]}) + # todo: doesn't play well with flattening + # th.ArrayType( + # th.ObjectType( + # th.Property("label", th.StringType), + # th.Property("required", th.BooleanType), + # th.Property("subscriptionTypeId", th.IntegerType), + # ) + # ) + ), + th.Property("consentToProcessText", th.StringType), + th.Property("privacyText", th.StringType), + th.Property("type", th.StringType), + )), ).to_dict() diff --git a/tap_hubspot/schemas/marketing/ListContacts.py b/tap_hubspot/schemas/marketing/ListContacts.py new file mode 100644 index 0000000..e3507f7 --- /dev/null +++ b/tap_hubspot/schemas/marketing/ListContacts.py @@ -0,0 +1,30 @@ +from singer_sdk import typing as th + +schema = th.PropertiesList( + th.Property("list_id", th.IntegerType), + th.Property("addedAt", th.IntegerType), + th.Property("vid", th.IntegerType), + th.Property("canonical-vid", th.IntegerType), + th.Property("merged-vids", th.ArrayType(th.ObjectType)), + th.Property("merge-audits", th.ArrayType(th.ObjectType)), + th.Property("portal-id", th.IntegerType), + th.Property("is-contact", th.BooleanType), + th.Property("properties", th.ObjectType()), + th.Property("form-submissions", th.ArrayType( + th.ObjectType + )), + th.Property("identity-profiles", th.ArrayType( + th.ObjectType( + th.Property("vid", th.IntegerType), + th.Property("saved-at-timestamp", th.IntegerType), + th.Property("deleted-changed-timestamp", th.IntegerType), + th.Property("identities", th.ArrayType( + th.ObjectType( + th.Property("type", th.StringType), + th.Property("value", th.StringType), + th.Property("timestamp", th.IntegerType), + ) + )), + ) + )), +).to_dict() diff --git a/tap_hubspot/schemas/marketing/Lists.py b/tap_hubspot/schemas/marketing/Lists.py new file mode 100644 index 0000000..78ce9c9 --- /dev/null +++ b/tap_hubspot/schemas/marketing/Lists.py @@ -0,0 +1,39 @@ +from singer_sdk import typing as th + +schema = th.PropertiesList( + th.Property("ListId", th.IntegerType), + th.Property("name", th.StringType), + th.Property("createAt", th.IntegerType), + th.Property("updatedAt", th.IntegerType), + th.Property("dynamic", th.BooleanType), + th.Property( + "filters", + th.ArrayType( + th.ArrayType( + th.ObjectType( + th.Property("filterFamily", th.StringType), + th.Property("withinTimeMode", th.StringType), + th.Property("checkPastVersions", th.BooleanType), + th.Property("type", th.StringType), + th.Property("property", th.StringType), + th.Property("value", th.StringType), + th.Property("operator", th.StringType), + ) + ) + ) + ), + th.Property( + "metaData", + th.ObjectType( + th.Property("processing", th.StringType), + th.Property("size", th.StringType), + th.Property("error", th.StringType), + th.Property("lastProcessingStateChangeAt", th.IntegerType), + th.Property("lastSizeChangeAt", th.IntegerType), + ) + ), + th.Property("portalId", th.IntegerType), + th.Property("listType", th.StringType), + th.Property("internalListId", th.IntegerType), + th.Property("deleteable", th.BooleanType) +).to_dict() diff --git a/tap_hubspot/tap.py b/tap_hubspot/tap.py index 3bf35e9..d9a559c 100644 --- a/tap_hubspot/tap.py +++ b/tap_hubspot/tap.py @@ -29,6 +29,8 @@ MarketingCampaignIdsStream, MarketingCampaignsStream, MarketingFormsStream, + MarketingListsStream, + MarketingListContactsStream, ) from tap_hubspot.events_streams import ( @@ -55,7 +57,6 @@ AssociationsDealsToContactsStream, ContactsStream, CompaniesStream, - ContactsStream, DealsStream, MeetingsStream, PropertiesCompaniesStream, @@ -68,6 +69,8 @@ MarketingCampaignIdsStream, MarketingCampaignsStream, MarketingFormsStream, + MarketingListsStream, + MarketingListContactsStream, # Events WebAnalyticsContactsStream, WebAnalyticsDealsStream, diff --git a/tap_hubspot/tests/test_core.py b/tap_hubspot/tests/test_core.py index 22ba91c..20209cf 100644 --- a/tap_hubspot/tests/test_core.py +++ b/tap_hubspot/tests/test_core.py @@ -7,16 +7,54 @@ from tap_hubspot.tap import TapHubspot SAMPLE_CONFIG = { - "start_date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d") + "access_token": "accesstoken", + "start_date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d"), } # Run standard built-in tap tests from the SDK: -def test_standard_tap_tests(): +def test_standard_tap_tests(requests_mock): """Run standard tap tests from the SDK.""" - tests = get_standard_tap_tests( - TapHubspot, - config=SAMPLE_CONFIG - ) + for stream in ["contacts", "companies", "deals", "meetings"]: + requests_mock.get( + f"https://api.hubapi.com/crm/v3/properties/{stream}", + json={"results": [{"name": "propertyname", "type": "propertytype"}]}, + ) + + streams1 = [ + "https://api.hubapi.com/analytics/v2/views?limit=100", + "https://api.hubapi.com/email/public/v1/campaigns/by-id?limit=100&orderBy=created", + "https://api.hubapi.com/crm/v3/objects/companies?limit=100&properties=propertyname&archived=True", + "https://api.hubapi.com/crm/v3/objects/companies?limit=100&properties=propertyname&archived=False", + "https://api.hubapi.com/crm/v3/objects/contacts?limit=100&properties=propertyname&archived=True", + "https://api.hubapi.com/crm/v3/objects/contacts?limit=100&properties=propertyname&archived=False", + "https://api.hubapi.com/crm/v3/objects/deals?limit=100&properties=propertyname&archived=True", + "https://api.hubapi.com/crm/v3/objects/deals?limit=100&properties=propertyname&archived=False", + "https://api.hubapi.com/marketing/v3/forms/?limit=100", + "https://api.hubapi.com/crm/v3/objects/meetings?limit=100&properties=propertyname", + "https://api.hubapi.com/crm/v3/owners?limit=100", + ] + for s in streams1: + requests_mock.get( + s, + json=[{"id": "1", "updatedDate": "2018-08-07"}], + ) + + streams2 = [ + "https://api.hubapi.com/marketing-emails/v1/emails/with-statistics?limit=100&offset=100&orderBy=created", + "https://api.hubapi.com/automation/v3/workflows?limit=100", + "https://api.hubapi.com/marketing/v3/forms/?count=100&formTypes=all", + "https://api.hubapi.com/contacts/v1/lists?limit=100&count=100&offset=0", + ] + for s in streams2: + requests_mock.get( + s, + json={ + "total": 1, + "objects": [{"updated": 1553538703608}], + "workflows": [{"updatedAt": 1467737836223}], + }, + ) + tests = get_standard_tap_tests(TapHubspot, config=SAMPLE_CONFIG) for test in tests: test()