diff --git a/Pipfile b/Pipfile index fa8fd56..e76c36b 100644 --- a/Pipfile +++ b/Pipfile @@ -16,7 +16,7 @@ pandas = "*" protobuf = "*" pyathena = "*" pyarrow = "*" -pylint = "<3.0.0" +pylint = "*" requests = "*" setuptools = "*" snowflake-connector-python = "*" diff --git a/Pipfile.lock b/Pipfile.lock index e5d5092..a16a2f6 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "fadabcccb48003d4f6b24ab066e9114d724f413a7475e84dca107f09f28f8476" + "sha256": "5767d4dcd989238a474680977cfd85808de205f446f321719e9142f4689aa1aa" }, "pipfile-spec": 6, "requires": { @@ -25,71 +25,79 @@ }, "astroid": { "hashes": [ - "sha256:1aa149fc5c6589e3d0ece885b4491acd80af4f087baafa3fb5203b113e68cd3c", - "sha256:6c107453dffee9055899705de3c9ead36e74119cee151e5a9aaf7f0b0e020a6a" + "sha256:16ee8ca5c75ac828783028cc1f967777f0e507c6886a295ad143e0f405b975a2", + "sha256:f7f829f8506ade59f1b3c6c93d8fac5b1ebc721685fa9af23e9794daf1d450a3" ], - "markers": "python_full_version >= '3.7.2'", - "version": "==2.15.8" + "markers": "python_full_version >= '3.8.0'", + "version": "==3.2.0" + }, + "backports.tarfile": { + "hashes": [ + "sha256:73e0179647803d3726d82e76089d01d8549ceca9bace469953fcb4d97cf2d417", + "sha256:9c2ef9696cb73374f7164e17fc761389393ca76777036f5aad42e8b93fcd8009" + ], + "markers": "python_version < '3.12'", + "version": "==1.1.1" }, "black": { "hashes": [ - "sha256:0808494f2b2df923ffc5723ed3c7b096bd76341f6213989759287611e9837d50", - "sha256:1fa88a0f74e50e4487477bc0bb900c6781dbddfdfa32691e780bf854c3b4a47f", - "sha256:25e57fd232a6d6ff3f4478a6fd0580838e47c93c83eaf1ccc92d4faf27112c4e", - "sha256:2d9e13db441c509a3763a7a3d9a49ccc1b4e974a47be4e08ade2a228876500ec", - "sha256:3e1b38b3135fd4c025c28c55ddfc236b05af657828a8a6abe5deec419a0b7055", - "sha256:3fa4be75ef2a6b96ea8d92b1587dd8cb3a35c7e3d51f0738ced0781c3aa3a5a3", - "sha256:4ce3ef14ebe8d9509188014d96af1c456a910d5b5cbf434a09fef7e024b3d0d5", - "sha256:4f0031eaa7b921db76decd73636ef3a12c942ed367d8c3841a0739412b260a54", - "sha256:602cfb1196dc692424c70b6507593a2b29aac0547c1be9a1d1365f0d964c353b", - "sha256:6d1bd9c210f8b109b1762ec9fd36592fdd528485aadb3f5849b2740ef17e674e", - "sha256:78baad24af0f033958cad29731e27363183e140962595def56423e626f4bee3e", - "sha256:8d4df77958a622f9b5a4c96edb4b8c0034f8434032ab11077ec6c56ae9f384ba", - "sha256:97e56155c6b737854e60a9ab1c598ff2533d57e7506d97af5481141671abf3ea", - "sha256:9c4352800f14be5b4864016882cdba10755bd50805c95f728011bcb47a4afd59", - "sha256:a4d6a9668e45ad99d2f8ec70d5c8c04ef4f32f648ef39048d010b0689832ec6d", - "sha256:a920b569dc6b3472513ba6ddea21f440d4b4c699494d2e972a1753cdc25df7b0", - "sha256:ae76c22bde5cbb6bfd211ec343ded2163bba7883c7bc77f6b756a1049436fbb9", - "sha256:b18fb2ae6c4bb63eebe5be6bd869ba2f14fd0259bda7d18a46b764d8fb86298a", - "sha256:c04b6d9d20e9c13f43eee8ea87d44156b8505ca8a3c878773f68b4e4812a421e", - "sha256:c88b3711d12905b74206227109272673edce0cb29f27e1385f33b0163c414bba", - "sha256:dd15245c8b68fe2b6bd0f32c1556509d11bb33aec9b5d0866dd8e2ed3dba09c2", - "sha256:e0aaf6041986767a5e0ce663c7a2f0e9eaf21e6ff87a5f95cbf3675bfd4c41d2" + "sha256:257d724c2c9b1660f353b36c802ccece186a30accc7742c176d29c146df6e474", + "sha256:37aae07b029fa0174d39daf02748b379399b909652a806e5708199bd93899da1", + "sha256:415e686e87dbbe6f4cd5ef0fbf764af7b89f9057b97c908742b6008cc554b9c0", + "sha256:48a85f2cb5e6799a9ef05347b476cce6c182d6c71ee36925a6c194d074336ef8", + "sha256:7768a0dbf16a39aa5e9a3ded568bb545c8c2727396d063bbaf847df05b08cd96", + "sha256:7e122b1c4fb252fd85df3ca93578732b4749d9be076593076ef4d07a0233c3e1", + "sha256:88c57dc656038f1ab9f92b3eb5335ee9b021412feaa46330d5eba4e51fe49b04", + "sha256:8e537d281831ad0e71007dcdcbe50a71470b978c453fa41ce77186bbe0ed6021", + "sha256:98e123f1d5cfd42f886624d84464f7756f60ff6eab89ae845210631714f6db94", + "sha256:accf49e151c8ed2c0cdc528691838afd217c50412534e876a19270fea1e28e2d", + "sha256:b1530ae42e9d6d5b670a34db49a94115a64596bc77710b1d05e9801e62ca0a7c", + "sha256:b9176b9832e84308818a99a561e90aa479e73c523b3f77afd07913380ae2eab7", + "sha256:bdde6f877a18f24844e381d45e9947a49e97933573ac9d4345399be37621e26c", + "sha256:be8bef99eb46d5021bf053114442914baeb3649a89dc5f3a555c88737e5e98fc", + "sha256:bf10f7310db693bb62692609b397e8d67257c55f949abde4c67f9cc574492cc7", + "sha256:c872b53057f000085da66a19c55d68f6f8ddcac2642392ad3a355878406fbd4d", + "sha256:d36ed1124bb81b32f8614555b34cc4259c3fbc7eec17870e8ff8ded335b58d8c", + "sha256:da33a1a5e49c4122ccdfd56cd021ff1ebc4a1ec4e2d01594fef9b6f267a9e741", + "sha256:dd1b5a14e417189db4c7b64a6540f31730713d173f0b63e55fabd52d61d8fdce", + "sha256:e151054aa00bad1f4e1f04919542885f89f5f7d086b8a59e5000e6c616896ffb", + "sha256:eaea3008c281f1038edb473c1aa8ed8143a5535ff18f978a318f10302b254063", + "sha256:ef703f83fc32e131e9bcc0a5094cfe85599e7109f896fe8bc96cc402f3eb4b6e" ], "index": "pypi", - "version": "==23.12.1" + "version": "==24.4.2" }, "boto3": { "hashes": [ - "sha256:5e38ca63007e903a7efe0a1751a0374d287b50d7bc148b9d3d495cdf74a0b712", - "sha256:ae7cfdf45f4dfd33bd3e84e36afcfbf0517e64a32e647989a068f34d053572b8" + "sha256:b633e8fbf7145bdb995ce68a27d096bb89fd393185b0e773418d81cd78db5a03", + "sha256:f2c11635be0de7b7c06eb606ece1add125e02d6ed521592294a0a21af09af135" ], "markers": "python_version >= '3.8'", - "version": "==1.34.18" + "version": "==1.34.105" }, "botocore": { "hashes": [ - "sha256:2067d8385c11b7cf2d336227d8fa5aea632fe61afbadb3168dc169dcc13d8c3e", - "sha256:85a77e72560a45b0dfdad94f92f5e114c82be07a51bb2d19dd310dab8be158cf" + "sha256:727d5d3e800ac8b705fca6e19b6fefa1e728a81d62a712df9bd32ed0117c740b", + "sha256:a459d060b541beecb50681e6e8a39313cca981e146a59ba7c5229d62f631a016" ], "markers": "python_version >= '3.8'", - "version": "==1.34.18" + "version": "==1.34.105" }, "cachetools": { "hashes": [ - "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2", - "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1" + "sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945", + "sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105" ], "markers": "python_version >= '3.7'", - "version": "==5.3.2" + "version": "==5.3.3" }, "certifi": { "hashes": [ - "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1", - "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474" + "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f", + "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1" ], "markers": "python_version >= '3.6'", - "version": "==2023.11.17" + "version": "==2024.2.2" }, "cffi": { "hashes": [ @@ -255,40 +263,49 @@ }, "cryptography": { "hashes": [ - "sha256:079b85658ea2f59c4f43b70f8119a52414cdb7be34da5d019a77bf96d473b960", - "sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a", - "sha256:13f93ce9bea8016c253b34afc6bd6a75993e5c40672ed5405a9c832f0d4a00bc", - "sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a", - "sha256:3c78451b78313fa81607fa1b3f1ae0a5ddd8014c38a02d9db0616133987b9cdf", - "sha256:43f2552a2378b44869fe8827aa19e69512e3245a219104438692385b0ee119d1", - "sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39", - "sha256:49f0805fc0b2ac8d4882dd52f4a3b935b210935d500b6b805f321addc8177406", - "sha256:5429ec739a29df2e29e15d082f1d9ad683701f0ec7709ca479b3ff2708dae65a", - "sha256:5a1b41bc97f1ad230a41657d9155113c7521953869ae57ac39ac7f1bb471469a", - "sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c", - "sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be", - "sha256:841df4caa01008bad253bce2a6f7b47f86dc9f08df4b433c404def869f590a15", - "sha256:90452ba79b8788fa380dfb587cca692976ef4e757b194b093d845e8d99f612f2", - "sha256:928258ba5d6f8ae644e764d0f996d61a8777559f72dfeb2eea7e2fe0ad6e782d", - "sha256:af03b32695b24d85a75d40e1ba39ffe7db7ffcb099fe507b39fd41a565f1b157", - "sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003", - "sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248", - "sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a", - "sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec", - "sha256:d6c391c021ab1f7a82da5d8d0b3cee2f4b2c455ec86c8aebbc84837a631ff309", - "sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7", - "sha256:f983596065a18a2183e7f79ab3fd4c475205b839e02cbc0efbbf9666c4b3083d" + "sha256:02c0eee2d7133bdbbc5e24441258d5d2244beb31da5ed19fbb80315f4bbbff55", + "sha256:0d563795db98b4cd57742a78a288cdbdc9daedac29f2239793071fe114f13785", + "sha256:16268d46086bb8ad5bf0a2b5544d8a9ed87a0e33f5e77dd3c3301e63d941a83b", + "sha256:1a58839984d9cb34c855197043eaae2c187d930ca6d644612843b4fe8513c886", + "sha256:2954fccea107026512b15afb4aa664a5640cd0af630e2ee3962f2602693f0c82", + "sha256:2e47577f9b18723fa294b0ea9a17d5e53a227867a0a4904a1a076d1646d45ca1", + "sha256:31adb7d06fe4383226c3e963471f6837742889b3c4caa55aac20ad951bc8ffda", + "sha256:3577d029bc3f4827dd5bf8bf7710cac13527b470bbf1820a3f394adb38ed7d5f", + "sha256:36017400817987670037fbb0324d71489b6ead6231c9604f8fc1f7d008087c68", + "sha256:362e7197754c231797ec45ee081f3088a27a47c6c01eff2ac83f60f85a50fe60", + "sha256:3de9a45d3b2b7d8088c3fbf1ed4395dfeff79d07842217b38df14ef09ce1d8d7", + "sha256:4f698edacf9c9e0371112792558d2f705b5645076cc0aaae02f816a0171770fd", + "sha256:5482e789294854c28237bba77c4c83be698be740e31a3ae5e879ee5444166582", + "sha256:5e44507bf8d14b36b8389b226665d597bc0f18ea035d75b4e53c7b1ea84583cc", + "sha256:779245e13b9a6638df14641d029add5dc17edbef6ec915688f3acb9e720a5858", + "sha256:789caea816c6704f63f6241a519bfa347f72fbd67ba28d04636b7c6b7da94b0b", + "sha256:7f8b25fa616d8b846aef64b15c606bb0828dbc35faf90566eb139aa9cff67af2", + "sha256:8cb8ce7c3347fcf9446f201dc30e2d5a3c898d009126010cbd1f443f28b52678", + "sha256:93a3209f6bb2b33e725ed08ee0991b92976dfdcf4e8b38646540674fc7508e13", + "sha256:a3a5ac8b56fe37f3125e5b72b61dcde43283e5370827f5233893d461b7360cd4", + "sha256:a47787a5e3649008a1102d3df55424e86606c9bae6fb77ac59afe06d234605f8", + "sha256:a79165431551042cc9d1d90e6145d5d0d3ab0f2d66326c201d9b0e7f5bf43604", + "sha256:a987f840718078212fdf4504d0fd4c6effe34a7e4740378e59d47696e8dfb477", + "sha256:a9bc127cdc4ecf87a5ea22a2556cab6c7eda2923f84e4f3cc588e8470ce4e42e", + "sha256:bd13b5e9b543532453de08bcdc3cc7cebec6f9883e886fd20a92f26940fd3e7a", + "sha256:c65f96dad14f8528a447414125e1fc8feb2ad5a272b8f68477abbcc1ea7d94b9", + "sha256:d8e3098721b84392ee45af2dd554c947c32cc52f862b6a3ae982dbb90f577f14", + "sha256:e6b79d0adb01aae87e8a44c2b64bc3f3fe59515280e00fb6d57a7267a2583cda", + "sha256:e6b8f1881dac458c34778d0a424ae5769de30544fc678eac51c1c8bb2183e9da", + "sha256:e9b2a6309f14c0497f348d08a065d52f3020656f675819fc405fb63bbcd26562", + "sha256:ecbfbc00bf55888edda9868a4cf927205de8499e7fabe6c050322298382953f2", + "sha256:efd0bf5205240182e0f13bcaea41be4fdf5c22c5129fc7ced4a0282ac86998c9" ], "markers": "python_version >= '3.7'", - "version": "==41.0.7" + "version": "==42.0.7" }, "dill": { "hashes": [ - "sha256:76b122c08ef4ce2eedcd4d1abd8e641114bfc6c2867f49f3c41facf65bf19f5e", - "sha256:cc1c8b182eb3013e24bd475ff2e9295af86c1a38eb1aff128dac8962a9ce3c03" + "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca", + "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7" ], "markers": "python_version < '3.11'", - "version": "==0.3.7" + "version": "==0.3.8" }, "docutils": { "hashes": [ @@ -308,43 +325,43 @@ }, "filelock": { "hashes": [ - "sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e", - "sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c" + "sha256:43339835842f110ca7ae60f1e1c160714c5a6afd15a2873419ab185334975c0f", + "sha256:6ea72da3be9b8c82afd3edcf99f2fffbb5076335a5ae4d03248bb5b6c3eae78a" ], "markers": "python_version >= '3.8'", - "version": "==3.13.1" + "version": "==3.14.0" }, "fsspec": { "hashes": [ - "sha256:8548d39e8810b59c38014934f6b31e57f40c1b20f911f4cc2b85389c7e9bf0cb", - "sha256:d800d87f72189a745fa3d6b033b9dc4a34ad069f60ca60b943a63599f5501960" + "sha256:1d021b0b0f933e3b3029ed808eb400c08ba101ca2de4b3483fbc9ca23fcee94a", + "sha256:e0fdbc446d67e182f49a70b82cf7889028a63588fde6b222521f10937b2b670c" ], "markers": "python_version >= '3.8'", - "version": "==2023.12.2" + "version": "==2024.5.0" }, "google-api-core": { "hashes": [ - "sha256:2aa56d2be495551e66bbff7f729b790546f87d5c90e74781aa77233bcb395a8a", - "sha256:abc978a72658f14a2df1e5e12532effe40f94f868f6e23d95133bd6abcca35ca" + "sha256:8661eec4078c35428fd3f69a2c7ee29e342896b70f01d1a1cbcb334372dd6251", + "sha256:cf1b7c2694047886d2af1128a03ae99e391108a08804f87cfd35970e49c9cd10" ], "index": "pypi", - "version": "==2.15.0" + "version": "==2.19.0" }, "google-auth": { "hashes": [ - "sha256:3f445c8ce9b61ed6459aad86d8ccdba4a9afed841b2d1451a11ef4db08957424", - "sha256:97327dbbf58cccb58fc5a1712bba403ae76668e64814eb30f7316f7e27126b81" + "sha256:672dff332d073227550ffc7457868ac4218d6c500b155fe6cc17d2b13602c360", + "sha256:d452ad095688cd52bae0ad6fafe027f6a6d6f560e810fec20914e17a09526415" ], "markers": "python_version >= '3.7'", - "version": "==2.26.2" + "version": "==2.29.0" }, "google-cloud-bigquery": { "hashes": [ - "sha256:1d6abf4b1d740df17cb43a078789872af8059a0b1dd999f32ea69ebc6f7ba7ef", - "sha256:8bac7754f92bf87ee81f38deabb7554d82bb9591fbe06a5c82f33e46e5a482f9" + "sha256:80c8e31a23b68b7d3ae5d138c9a9edff69d100ee812db73a5e63c79a13a5063d", + "sha256:957591e6f948d7cb4aa0f7a8e4e47b4617cd7f0269e28a71c37953c39b6e8a4c" ], "index": "pypi", - "version": "==3.16.0" + "version": "==3.22.0" }, "google-cloud-core": { "hashes": [ @@ -438,11 +455,11 @@ }, "googleapis-common-protos": { "hashes": [ - "sha256:4750113612205514f9f6aa4cb00d523a94f3e8c06c5ad2fee466387dc4875f07", - "sha256:83f0ece9f94e5672cced82f592d2a5edf527a96ed1794f0bab36d5735c996277" + "sha256:17ad01b11d5f1d0171c06d3ba5c04c54474e883b66b949722b4938ee2694ef4e", + "sha256:ae45f75702f7c08b541f750854a678bd8f534a1a6bace6afe975f1d0a82d6632" ], "index": "pypi", - "version": "==1.62.0" + "version": "==1.63.0" }, "greenlet": { "hashes": [ @@ -510,87 +527,86 @@ }, "grpcio": { "hashes": [ - "sha256:073f959c6f570797272f4ee9464a9997eaf1e98c27cb680225b82b53390d61e6", - "sha256:0fd3b3968ffe7643144580f260f04d39d869fcc2cddb745deef078b09fd2b328", - "sha256:1434ca77d6fed4ea312901122dc8da6c4389738bf5788f43efb19a838ac03ead", - "sha256:1c30bb23a41df95109db130a6cc1b974844300ae2e5d68dd4947aacba5985aa5", - "sha256:20e7a4f7ded59097c84059d28230907cd97130fa74f4a8bfd1d8e5ba18c81491", - "sha256:2199165a1affb666aa24adf0c97436686d0a61bc5fc113c037701fb7c7fceb96", - "sha256:297eef542156d6b15174a1231c2493ea9ea54af8d016b8ca7d5d9cc65cfcc444", - "sha256:2aef56e85901c2397bd557c5ba514f84de1f0ae5dd132f5d5fed042858115951", - "sha256:30943b9530fe3620e3b195c03130396cd0ee3a0d10a66c1bee715d1819001eaf", - "sha256:3b36a2c6d4920ba88fa98075fdd58ff94ebeb8acc1215ae07d01a418af4c0253", - "sha256:428d699c8553c27e98f4d29fdc0f0edc50e9a8a7590bfd294d2edb0da7be3629", - "sha256:43e636dc2ce9ece583b3e2ca41df5c983f4302eabc6d5f9cd04f0562ee8ec1ae", - "sha256:452ca5b4afed30e7274445dd9b441a35ece656ec1600b77fff8c216fdf07df43", - "sha256:467a7d31554892eed2aa6c2d47ded1079fc40ea0b9601d9f79204afa8902274b", - "sha256:4b44d7e39964e808b071714666a812049765b26b3ea48c4434a3b317bac82f14", - "sha256:4c86343cf9ff7b2514dd229bdd88ebba760bd8973dac192ae687ff75e39ebfab", - "sha256:5208a57eae445ae84a219dfd8b56e04313445d146873117b5fa75f3245bc1390", - "sha256:5ff21e000ff2f658430bde5288cb1ac440ff15c0d7d18b5fb222f941b46cb0d2", - "sha256:675997222f2e2f22928fbba640824aebd43791116034f62006e19730715166c0", - "sha256:676e4a44e740deaba0f4d95ba1d8c5c89a2fcc43d02c39f69450b1fa19d39590", - "sha256:6e306b97966369b889985a562ede9d99180def39ad42c8014628dd3cc343f508", - "sha256:6fd9584bf1bccdfff1512719316efa77be235469e1e3295dce64538c4773840b", - "sha256:705a68a973c4c76db5d369ed573fec3367d7d196673fa86614b33d8c8e9ebb08", - "sha256:74d7d9fa97809c5b892449b28a65ec2bfa458a4735ddad46074f9f7d9550ad13", - "sha256:77c8a317f0fd5a0a2be8ed5cbe5341537d5c00bb79b3bb27ba7c5378ba77dbca", - "sha256:79a050889eb8d57a93ed21d9585bb63fca881666fc709f5d9f7f9372f5e7fd03", - "sha256:7db16dd4ea1b05ada504f08d0dca1cd9b926bed3770f50e715d087c6f00ad748", - "sha256:83f2292ae292ed5a47cdcb9821039ca8e88902923198f2193f13959360c01860", - "sha256:87c9224acba0ad8bacddf427a1c2772e17ce50b3042a789547af27099c5f751d", - "sha256:8a97a681e82bc11a42d4372fe57898d270a2707f36c45c6676e49ce0d5c41353", - "sha256:9073513ec380434eb8d21970e1ab3161041de121f4018bbed3146839451a6d8e", - "sha256:90bdd76b3f04bdb21de5398b8a7c629676c81dfac290f5f19883857e9371d28c", - "sha256:91229d7203f1ef0ab420c9b53fe2ca5c1fbeb34f69b3bc1b5089466237a4a134", - "sha256:92f88ca1b956eb8427a11bb8b4a0c0b2b03377235fc5102cb05e533b8693a415", - "sha256:95ae3e8e2c1b9bf671817f86f155c5da7d49a2289c5cf27a319458c3e025c320", - "sha256:9e30be89a75ee66aec7f9e60086fadb37ff8c0ba49a022887c28c134341f7179", - "sha256:a48edde788b99214613e440fce495bbe2b1e142a7f214cce9e0832146c41e324", - "sha256:a7152fa6e597c20cb97923407cf0934e14224af42c2b8d915f48bc3ad2d9ac18", - "sha256:a9c7b71211f066908e518a2ef7a5e211670761651039f0d6a80d8d40054047df", - "sha256:b0571a5aef36ba9177e262dc88a9240c866d903a62799e44fd4aae3f9a2ec17e", - "sha256:b0fb2d4801546598ac5cd18e3ec79c1a9af8b8f2a86283c55a5337c5aeca4b1b", - "sha256:b10241250cb77657ab315270b064a6c7f1add58af94befa20687e7c8d8603ae6", - "sha256:b87efe4a380887425bb15f220079aa8336276398dc33fce38c64d278164f963d", - "sha256:b98f43fcdb16172dec5f4b49f2fece4b16a99fd284d81c6bbac1b3b69fcbe0ff", - "sha256:c193109ca4070cdcaa6eff00fdb5a56233dc7610216d58fb81638f89f02e4968", - "sha256:c826f93050c73e7769806f92e601e0efdb83ec8d7c76ddf45d514fee54e8e619", - "sha256:d020cfa595d1f8f5c6b343530cd3ca16ae5aefdd1e832b777f9f0eb105f5b139", - "sha256:d6a478581b1a1a8fdf3318ecb5f4d0cda41cacdffe2b527c23707c9c1b8fdb55", - "sha256:de2ad69c9a094bf37c1102b5744c9aec6cf74d2b635558b779085d0263166454", - "sha256:e278eafb406f7e1b1b637c2cf51d3ad45883bb5bd1ca56bc05e4fc135dfdaa65", - "sha256:e381fe0c2aa6c03b056ad8f52f8efca7be29fb4d9ae2f8873520843b6039612a", - "sha256:e61e76020e0c332a98290323ecfec721c9544f5b739fab925b6e8cbe1944cf19", - "sha256:f897c3b127532e6befdcf961c415c97f320d45614daf84deba0a54e64ea2457b", - "sha256:fb464479934778d7cc5baf463d959d361954d6533ad34c3a4f1d267e86ee25fd" + "sha256:01799e8649f9e94ba7db1aeb3452188048b0019dc37696b0f5ce212c87c560c3", + "sha256:0697563d1d84d6985e40ec5ec596ff41b52abb3fd91ec240e8cb44a63b895094", + "sha256:08e1559fd3b3b4468486b26b0af64a3904a8dbc78d8d936af9c1cf9636eb3e8b", + "sha256:166e5c460e5d7d4656ff9e63b13e1f6029b122104c1633d5f37eaea348d7356d", + "sha256:1ff737cf29b5b801619f10e59b581869e32f400159e8b12d7a97e7e3bdeee6a2", + "sha256:219bb1848cd2c90348c79ed0a6b0ea51866bc7e72fa6e205e459fedab5770172", + "sha256:259e11932230d70ef24a21b9fb5bb947eb4703f57865a404054400ee92f42f5d", + "sha256:2e93aca840c29d4ab5db93f94ed0a0ca899e241f2e8aec6334ab3575dc46125c", + "sha256:3a6d1f9ea965e750db7b4ee6f9fdef5fdf135abe8a249e75d84b0a3e0c668a1b", + "sha256:50344663068041b34a992c19c600236e7abb42d6ec32567916b87b4c8b8833b3", + "sha256:56cdf96ff82e3cc90dbe8bac260352993f23e8e256e063c327b6cf9c88daf7a9", + "sha256:5c039ef01516039fa39da8a8a43a95b64e288f79f42a17e6c2904a02a319b357", + "sha256:6426e1fb92d006e47476d42b8f240c1d916a6d4423c5258ccc5b105e43438f61", + "sha256:65bf975639a1f93bee63ca60d2e4951f1b543f498d581869922910a476ead2f5", + "sha256:6a1a3642d76f887aa4009d92f71eb37809abceb3b7b5a1eec9c554a246f20e3a", + "sha256:6ef0ad92873672a2a3767cb827b64741c363ebaa27e7f21659e4e31f4d750280", + "sha256:756fed02dacd24e8f488f295a913f250b56b98fb793f41d5b2de6c44fb762434", + "sha256:75f701ff645858a2b16bc8c9fc68af215a8bb2d5a9b647448129de6e85d52bce", + "sha256:8064d986d3a64ba21e498b9a376cbc5d6ab2e8ab0e288d39f266f0fca169b90d", + "sha256:878b1d88d0137df60e6b09b74cdb73db123f9579232c8456f53e9abc4f62eb3c", + "sha256:8f3f6883ce54a7a5f47db43289a0a4c776487912de1a0e2cc83fdaec9685cc9f", + "sha256:91b73d3f1340fefa1e1716c8c1ec9930c676d6b10a3513ab6c26004cb02d8b3f", + "sha256:93a46794cc96c3a674cdfb59ef9ce84d46185fe9421baf2268ccb556f8f81f57", + "sha256:93f45f27f516548e23e4ec3fbab21b060416007dbe768a111fc4611464cc773f", + "sha256:9e350cb096e5c67832e9b6e018cf8a0d2a53b2a958f6251615173165269a91b0", + "sha256:a2d60cd1d58817bc5985fae6168d8b5655c4981d448d0f5b6194bbcc038090d2", + "sha256:a3abfe0b0f6798dedd2e9e92e881d9acd0fdb62ae27dcbbfa7654a57e24060c0", + "sha256:a44624aad77bf8ca198c55af811fd28f2b3eaf0a50ec5b57b06c034416ef2d0a", + "sha256:a7b19dfc74d0be7032ca1eda0ed545e582ee46cd65c162f9e9fc6b26ef827dc6", + "sha256:ad2ac8903b2eae071055a927ef74121ed52d69468e91d9bcbd028bd0e554be6d", + "sha256:b005292369d9c1f80bf70c1db1c17c6c342da7576f1c689e8eee4fb0c256af85", + "sha256:b2e44f59316716532a993ca2966636df6fbe7be4ab6f099de6815570ebe4383a", + "sha256:b3afbd9d6827fa6f475a4f91db55e441113f6d3eb9b7ebb8fb806e5bb6d6bd0d", + "sha256:b416252ac5588d9dfb8a30a191451adbf534e9ce5f56bb02cd193f12d8845b7f", + "sha256:b5194775fec7dc3dbd6a935102bb156cd2c35efe1685b0a46c67b927c74f0cfb", + "sha256:cacdef0348a08e475a721967f48206a2254a1b26ee7637638d9e081761a5ba86", + "sha256:cd1e68776262dd44dedd7381b1a0ad09d9930ffb405f737d64f505eb7f77d6c7", + "sha256:cdcda1156dcc41e042d1e899ba1f5c2e9f3cd7625b3d6ebfa619806a4c1aadda", + "sha256:cf8dae9cc0412cb86c8de5a8f3be395c5119a370f3ce2e69c8b7d46bb9872c8d", + "sha256:d2497769895bb03efe3187fb1888fc20e98a5f18b3d14b606167dacda5789434", + "sha256:e3b77eaefc74d7eb861d3ffbdf91b50a1bb1639514ebe764c47773b833fa2d91", + "sha256:e48cee31bc5f5a31fb2f3b573764bd563aaa5472342860edcc7039525b53e46a", + "sha256:e4cbb2100ee46d024c45920d16e888ee5d3cf47c66e316210bc236d5bebc42b3", + "sha256:f28f8b2db7b86c77916829d64ab21ff49a9d8289ea1564a2b2a3a8ed9ffcccd3", + "sha256:f3023e14805c61bc439fb40ca545ac3d5740ce66120a678a3c6c2c55b70343d1", + "sha256:fdf348ae69c6ff484402cfdb14e18c1b0054ac2420079d575c53a60b9b2853ae" ], "index": "pypi", - "version": "==1.60.0" + "version": "==1.63.0" + }, + "grpcio-status": { + "hashes": [ + "sha256:206ddf0eb36bc99b033f03b2c8e95d319f0044defae9b41ae21408e7e0cda48f", + "sha256:62e1bfcb02025a1cd73732a2d33672d3e9d0df4d21c12c51e0bbcaf09bab742a" + ], + "version": "==1.62.2" }, "idna": { "hashes": [ - "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca", - "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f" + "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc", + "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0" ], "markers": "python_version >= '3.5'", - "version": "==3.6" + "version": "==3.7" }, "importlib-metadata": { "hashes": [ - "sha256:4805911c3a4ec7c3966410053e9ec6a1fecd629117df5adee56dfc9432a1081e", - "sha256:f238736bb06590ae52ac1fab06a3a9ef1d8dce2b7a35b5ab329371d6c8f5d2cc" + "sha256:30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570", + "sha256:b78938b926ee8d5f020fc4772d487045805a55ddbad2ecf21c6d60938dc7fcd2" ], "markers": "python_version >= '3.8'", - "version": "==7.0.1" + "version": "==7.1.0" }, "importlib-resources": { "hashes": [ - "sha256:3893a00122eafde6894c59914446a512f728a0c1a45f9bb9b63721b6bacf0b4a", - "sha256:e8bf90d8213b486f428c9c39714b920041cb02c184686a3dee24905aaa8105d6" + "sha256:50d10f043df931902d4194ea07ec57960f66a80449ff867bfe782b4c486ba78c", + "sha256:cdb2b453b8046ca4e3798eb1d84f3cce1446a0e8e7b5ef4efb600f19fc398145" ], "markers": "python_version < '3.9'", - "version": "==6.1.1" + "version": "==6.4.0" }, "isort": { "hashes": [ @@ -602,11 +618,27 @@ }, "jaraco.classes": { "hashes": [ - "sha256:10afa92b6743f25c0cf5f37c6bb6e18e2c5bb84a16527ccfc0040ea377e7aaeb", - "sha256:c063dd08e89217cee02c8d5e5ec560f2c8ce6cdc2fcdc2e68f7b2e5547ed3621" + "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd", + "sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790" + ], + "markers": "python_version >= '3.8'", + "version": "==3.4.0" + }, + "jaraco.context": { + "hashes": [ + "sha256:3e16388f7da43d384a1a7cd3452e72e14732ac9fe459678773a3608a812bf266", + "sha256:c2f67165ce1f9be20f32f650f25d8edfc1646a8aeee48ae06fb35f90763576d2" ], "markers": "python_version >= '3.8'", - "version": "==3.3.0" + "version": "==5.3.0" + }, + "jaraco.functools": { + "hashes": [ + "sha256:3b24ccb921d6b593bdceb56ce14799204f473976e2a9d4b15b04d0f2c2326664", + "sha256:d33fa765374c0611b52f8b3a795f8900869aa88c84769d4d1746cd68fb28c3e8" + ], + "markers": "python_version >= '3.8'", + "version": "==4.0.1" }, "jeepney": { "hashes": [ @@ -626,54 +658,11 @@ }, "keyring": { "hashes": [ - "sha256:4446d35d636e6a10b8bce7caa66913dd9eca5fd222ca03a3d42c38608ac30836", - "sha256:e730ecffd309658a08ee82535a3b5ec4b4c8669a9be11efb66249d8e0aeb9a25" + "sha256:2458681cdefc0dbc0b7eb6cf75d0b98e59f9ad9b2d4edd319d18f68bdca95e50", + "sha256:daaffd42dbda25ddafb1ad5fec4024e5bbcfe424597ca1ca452b299861e49f1b" ], "markers": "python_version >= '3.8'", - "version": "==24.3.0" - }, - "lazy-object-proxy": { - "hashes": [ - "sha256:009e6bb1f1935a62889ddc8541514b6a9e1fcf302667dcb049a0be5c8f613e56", - "sha256:02c83f957782cbbe8136bee26416686a6ae998c7b6191711a04da776dc9e47d4", - "sha256:0aefc7591920bbd360d57ea03c995cebc204b424524a5bd78406f6e1b8b2a5d8", - "sha256:127a789c75151db6af398b8972178afe6bda7d6f68730c057fbbc2e96b08d282", - "sha256:18dd842b49456aaa9a7cf535b04ca4571a302ff72ed8740d06b5adcd41fe0757", - "sha256:217138197c170a2a74ca0e05bddcd5f1796c735c37d0eee33e43259b192aa424", - "sha256:2297f08f08a2bb0d32a4265e98a006643cd7233fb7983032bd61ac7a02956b3b", - "sha256:2fc0a92c02fa1ca1e84fc60fa258458e5bf89d90a1ddaeb8ed9cc3147f417255", - "sha256:30b339b2a743c5288405aa79a69e706a06e02958eab31859f7f3c04980853b70", - "sha256:366c32fe5355ef5fc8a232c5436f4cc66e9d3e8967c01fb2e6302fd6627e3d94", - "sha256:3ad54b9ddbe20ae9f7c1b29e52f123120772b06dbb18ec6be9101369d63a4074", - "sha256:5ad9e6ed739285919aa9661a5bbed0aaf410aa60231373c5579c6b4801bd883c", - "sha256:5faf03a7d8942bb4476e3b62fd0f4cf94eaf4618e304a19865abf89a35c0bbee", - "sha256:75fc59fc450050b1b3c203c35020bc41bd2695ed692a392924c6ce180c6f1dc9", - "sha256:76a095cfe6045c7d0ca77db9934e8f7b71b14645f0094ffcd842349ada5c5fb9", - "sha256:78247b6d45f43a52ef35c25b5581459e85117225408a4128a3daf8bf9648ac69", - "sha256:782e2c9b2aab1708ffb07d4bf377d12901d7a1d99e5e410d648d892f8967ab1f", - "sha256:7ab7004cf2e59f7c2e4345604a3e6ea0d92ac44e1c2375527d56492014e690c3", - "sha256:80b39d3a151309efc8cc48675918891b865bdf742a8616a337cb0090791a0de9", - "sha256:80fa48bd89c8f2f456fc0765c11c23bf5af827febacd2f523ca5bc1893fcc09d", - "sha256:855e068b0358ab916454464a884779c7ffa312b8925c6f7401e952dcf3b89977", - "sha256:92f09ff65ecff3108e56526f9e2481b8116c0b9e1425325e13245abfd79bdb1b", - "sha256:952c81d415b9b80ea261d2372d2a4a2332a3890c2b83e0535f263ddfe43f0d43", - "sha256:9a3a87cf1e133e5b1994144c12ca4aa3d9698517fe1e2ca82977781b16955658", - "sha256:9e4ed0518a14dd26092614412936920ad081a424bdcb54cc13349a8e2c6d106a", - "sha256:a899b10e17743683b293a729d3a11f2f399e8a90c73b089e29f5d0fe3509f0dd", - "sha256:b1f711e2c6dcd4edd372cf5dec5c5a30d23bba06ee012093267b3376c079ec83", - "sha256:b4f87d4ed9064b2628da63830986c3d2dca7501e6018347798313fcf028e2fd4", - "sha256:cb73507defd385b7705c599a94474b1d5222a508e502553ef94114a143ec6696", - "sha256:dc0d2fc424e54c70c4bc06787e4072c4f3b1aa2f897dfdc34ce1013cf3ceef05", - "sha256:e221060b701e2aa2ea991542900dd13907a5c90fa80e199dbf5a03359019e7a3", - "sha256:e271058822765ad5e3bca7f05f2ace0de58a3f4e62045a8c90a0dfd2f8ad8cc6", - "sha256:e2adb09778797da09d2b5ebdbceebf7dd32e2c96f79da9052b2e87b6ea495895", - "sha256:e333e2324307a7b5d86adfa835bb500ee70bfcd1447384a822e96495796b0ca4", - "sha256:e98c8af98d5707dcdecc9ab0863c0ea6e88545d42ca7c3feffb6b4d1e370c7ba", - "sha256:edb45bb8278574710e68a6b021599a10ce730d156e5b254941754a9cc0b17d03", - "sha256:fec03caabbc6b59ea4a638bee5fce7117be8e99a4103d9d5ad77f15d6f81020c" - ], - "markers": "python_version >= '3.8'", - "version": "==1.10.0" + "version": "==25.2.1" }, "markdown-it-py": { "hashes": [ @@ -709,36 +698,36 @@ }, "mypy": { "hashes": [ - "sha256:028cf9f2cae89e202d7b6593cd98db6759379f17a319b5faf4f9978d7084cdc6", - "sha256:2afecd6354bbfb6e0160f4e4ad9ba6e4e003b767dd80d85516e71f2e955ab50d", - "sha256:2b5b6c721bd4aabaadead3a5e6fa85c11c6c795e0c81a7215776ef8afc66de02", - "sha256:42419861b43e6962a649068a61f4a4839205a3ef525b858377a960b9e2de6e0d", - "sha256:42c6680d256ab35637ef88891c6bd02514ccb7e1122133ac96055ff458f93fc3", - "sha256:485a8942f671120f76afffff70f259e1cd0f0cfe08f81c05d8816d958d4577d3", - "sha256:4c886c6cce2d070bd7df4ec4a05a13ee20c0aa60cb587e8d1265b6c03cf91da3", - "sha256:4e6d97288757e1ddba10dd9549ac27982e3e74a49d8d0179fc14d4365c7add66", - "sha256:4ef4be7baf08a203170f29e89d79064463b7fc7a0908b9d0d5114e8009c3a259", - "sha256:51720c776d148bad2372ca21ca29256ed483aa9a4cdefefcef49006dff2a6835", - "sha256:52825b01f5c4c1c4eb0db253ec09c7aa17e1a7304d247c48b6f3599ef40db8bd", - "sha256:538fd81bb5e430cc1381a443971c0475582ff9f434c16cd46d2c66763ce85d9d", - "sha256:5c1538c38584029352878a0466f03a8ee7547d7bd9f641f57a0f3017a7c905b8", - "sha256:6ff8b244d7085a0b425b56d327b480c3b29cafbd2eff27316a004f9a7391ae07", - "sha256:7178def594014aa6c35a8ff411cf37d682f428b3b5617ca79029d8ae72f5402b", - "sha256:720a5ca70e136b675af3af63db533c1c8c9181314d207568bbe79051f122669e", - "sha256:7f1478736fcebb90f97e40aff11a5f253af890c845ee0c850fe80aa060a267c6", - "sha256:855fe27b80375e5c5878492f0729540db47b186509c98dae341254c8f45f42ae", - "sha256:8963b83d53ee733a6e4196954502b33567ad07dfd74851f32be18eb932fb1cb9", - "sha256:9261ed810972061388918c83c3f5cd46079d875026ba97380f3e3978a72f503d", - "sha256:99b00bc72855812a60d253420d8a2eae839b0afa4938f09f4d2aa9bb4654263a", - "sha256:ab3c84fa13c04aeeeabb2a7f67a25ef5d77ac9d6486ff33ded762ef353aa5592", - "sha256:afe3fe972c645b4632c563d3f3eff1cdca2fa058f730df2b93a35e3b0c538218", - "sha256:d19c413b3c07cbecf1f991e2221746b0d2a9410b59cb3f4fb9557f0365a1a817", - "sha256:df9824ac11deaf007443e7ed2a4a26bebff98d2bc43c6da21b2b64185da011c4", - "sha256:e46f44b54ebddbeedbd3d5b289a893219065ef805d95094d16a0af6630f5d410", - "sha256:f5ac9a4eeb1ec0f1ccdc6f326bcdb464de5f80eb07fb38b5ddd7b0de6bc61e55" + "sha256:075cbf81f3e134eadaf247de187bd604748171d6b79736fa9b6c9685b4083061", + "sha256:12b6bfc1b1a66095ab413160a6e520e1dc076a28f3e22f7fb25ba3b000b4ef99", + "sha256:1ec404a7cbe9fc0e92cb0e67f55ce0c025014e26d33e54d9e506a0f2d07fe5de", + "sha256:28d0e038361b45f099cc086d9dd99c15ff14d0188f44ac883010e172ce86c38a", + "sha256:2b0695d605ddcd3eb2f736cd8b4e388288c21e7de85001e9f85df9187f2b50f9", + "sha256:3236a4c8f535a0631f85f5fcdffba71c7feeef76a6002fcba7c1a8e57c8be1ec", + "sha256:3be66771aa5c97602f382230165b856c231d1277c511c9a8dd058be4784472e1", + "sha256:3d087fcbec056c4ee34974da493a826ce316947485cef3901f511848e687c131", + "sha256:3f298531bca95ff615b6e9f2fc0333aae27fa48052903a0ac90215021cdcfa4f", + "sha256:4a2b5cdbb5dd35aa08ea9114436e0d79aceb2f38e32c21684dcf8e24e1e92821", + "sha256:4cf18f9d0efa1b16478c4c129eabec36148032575391095f73cae2e722fcf9d5", + "sha256:8b2cbaca148d0754a54d44121b5825ae71868c7592a53b7292eeb0f3fdae95ee", + "sha256:8f55583b12156c399dce2df7d16f8a5095291354f1e839c252ec6c0611e86e2e", + "sha256:92f93b21c0fe73dc00abf91022234c79d793318b8a96faac147cd579c1671746", + "sha256:9e36fb078cce9904c7989b9693e41cb9711e0600139ce3970c6ef814b6ebc2b2", + "sha256:9fd50226364cd2737351c79807775136b0abe084433b55b2e29181a4c3c878c0", + "sha256:a781f6ad4bab20eef8b65174a57e5203f4be627b46291f4589879bf4e257b97b", + "sha256:a87dbfa85971e8d59c9cc1fcf534efe664d8949e4c0b6b44e8ca548e746a8d53", + "sha256:b808e12113505b97d9023b0b5e0c0705a90571c6feefc6f215c1df9381256e30", + "sha256:bc6ac273b23c6b82da3bb25f4136c4fd42665f17f2cd850771cb600bdd2ebeda", + "sha256:cd777b780312ddb135bceb9bc8722a73ec95e042f911cc279e2ec3c667076051", + "sha256:da1cbf08fb3b851ab3b9523a884c232774008267b1f83371ace57f412fe308c2", + "sha256:e22e1527dc3d4aa94311d246b59e47f6455b8729f4968765ac1eacf9a4760bc7", + "sha256:f8c083976eb530019175aabadb60921e73b4f45736760826aa1689dda8208aee", + "sha256:f90cff89eea89273727d8783fef5d4a934be2fdca11b47def50cf5d311aff727", + "sha256:fa7ef5244615a2523b56c034becde4e9e3f9b034854c93639adb667ec9ec2976", + "sha256:fcfc70599efde5c67862a07a1aaf50e55bce629ace26bb19dc17cece5dd31ca4" ], "index": "pypi", - "version": "==1.8.0" + "version": "==1.10.0" }, "mypy-extensions": { "hashes": [ @@ -750,24 +739,24 @@ }, "nh3": { "hashes": [ - "sha256:0d02d0ff79dfd8208ed25a39c12cbda092388fff7f1662466e27d97ad011b770", - "sha256:3277481293b868b2715907310c7be0f1b9d10491d5adf9fce11756a97e97eddf", - "sha256:3b803a5875e7234907f7d64777dfde2b93db992376f3d6d7af7f3bc347deb305", - "sha256:427fecbb1031db085eaac9931362adf4a796428ef0163070c484b5a768e71601", - "sha256:5f0d77272ce6d34db6c87b4f894f037d55183d9518f948bba236fe81e2bb4e28", - "sha256:60684857cfa8fdbb74daa867e5cad3f0c9789415aba660614fe16cd66cbb9ec7", - "sha256:6f42f99f0cf6312e470b6c09e04da31f9abaadcd3eb591d7d1a88ea931dca7f3", - "sha256:86e447a63ca0b16318deb62498db4f76fc60699ce0a1231262880b38b6cff911", - "sha256:8d595df02413aa38586c24811237e95937ef18304e108b7e92c890a06793e3bf", - "sha256:9c0d415f6b7f2338f93035bba5c0d8c1b464e538bfbb1d598acd47d7969284f0", - "sha256:a5167a6403d19c515217b6bcaaa9be420974a6ac30e0da9e84d4fc67a5d474c5", - "sha256:ac19c0d68cd42ecd7ead91a3a032fdfff23d29302dbb1311e641a130dfefba97", - "sha256:b1e97221cedaf15a54f5243f2c5894bb12ca951ae4ddfd02a9d4ea9df9e1a29d", - "sha256:bc2d086fb540d0fa52ce35afaded4ea526b8fc4d3339f783db55c95de40ef02e", - "sha256:d1e30ff2d8d58fb2a14961f7aac1bbb1c51f9bdd7da727be35c63826060b0bf3", - "sha256:f3b53ba93bb7725acab1e030bc2ecd012a817040fd7851b332f86e2f9bb98dc6" - ], - "version": "==0.2.15" + "sha256:0316c25b76289cf23be6b66c77d3608a4fdf537b35426280032f432f14291b9a", + "sha256:1a814dd7bba1cb0aba5bcb9bebcc88fd801b63e21e2450ae6c52d3b3336bc911", + "sha256:1aa52a7def528297f256de0844e8dd680ee279e79583c76d6fa73a978186ddfb", + "sha256:22c26e20acbb253a5bdd33d432a326d18508a910e4dcf9a3316179860d53345a", + "sha256:40015514022af31975c0b3bca4014634fa13cb5dc4dbcbc00570acc781316dcc", + "sha256:40d0741a19c3d645e54efba71cb0d8c475b59135c1e3c580f879ad5514cbf028", + "sha256:551672fd71d06cd828e282abdb810d1be24e1abb7ae2543a8fa36a71c1006fe9", + "sha256:66f17d78826096291bd264f260213d2b3905e3c7fae6dfc5337d49429f1dc9f3", + "sha256:85cdbcca8ef10733bd31f931956f7fbb85145a4d11ab9e6742bbf44d88b7e351", + "sha256:a3f55fabe29164ba6026b5ad5c3151c314d136fd67415a17660b4aaddacf1b10", + "sha256:b4427ef0d2dfdec10b641ed0bdaf17957eb625b2ec0ea9329b3d28806c153d71", + "sha256:ba73a2f8d3a1b966e9cdba7b211779ad8a2561d2dba9674b8a19ed817923f65f", + "sha256:c21bac1a7245cbd88c0b0e4a420221b7bfa838a2814ee5bb924e9c2f10a1120b", + "sha256:c551eb2a3876e8ff2ac63dff1585236ed5dfec5ffd82216a7a174f7c5082a78a", + "sha256:c790769152308421283679a142dbdb3d1c46c79c823008ecea8e8141db1a2062", + "sha256:d7a25fd8c86657f5d9d576268e3b3767c5cd4f42867c9383618be8517f0f022a" + ], + "version": "==0.2.17" }, "numpy": { "hashes": [ @@ -813,11 +802,11 @@ }, "packaging": { "hashes": [ - "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5", - "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7" + "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", + "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9" ], "markers": "python_version >= '3.7'", - "version": "==23.2" + "version": "==24.0" }, "pandas": { "hashes": [ @@ -860,117 +849,126 @@ }, "pkginfo": { "hashes": [ - "sha256:4b7a555a6d5a22169fcc9cf7bfd78d296b0361adad412a346c1226849af5e546", - "sha256:8fd5896e8718a4372f0ea9cc9d96f6417c9b986e23a4d116dda26b62cc29d046" + "sha256:5df73835398d10db79f8eecd5cd86b1f6d29317589ea70796994d49399af6297", + "sha256:889a6da2ed7ffc58ab5b900d888ddce90bce912f2d2de1dc1c26f4cb9fe65097" ], "markers": "python_version >= '3.6'", - "version": "==1.9.6" + "version": "==1.10.0" }, "platformdirs": { "hashes": [ - "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3", - "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e" + "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee", + "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3" ], - "markers": "python_version >= '3.7'", - "version": "==3.11.0" + "markers": "python_version >= '3.8'", + "version": "==4.2.2" + }, + "proto-plus": { + "hashes": [ + "sha256:89075171ef11988b3fa157f5dbd8b9cf09d65fffee97e29ce403cd8defba19d2", + "sha256:a829c79e619e1cf632de091013a4173deed13a55f326ef84f05af6f50ff4c82c" + ], + "markers": "python_version >= '3.6'", + "version": "==1.23.0" }, "protobuf": { "hashes": [ - "sha256:10894a2885b7175d3984f2be8d9850712c57d5e7587a2410720af8be56cdaf62", - "sha256:2db9f8fa64fbdcdc93767d3cf81e0f2aef176284071507e3ede160811502fd3d", - "sha256:33a1aeef4b1927431d1be780e87b641e322b88d654203a9e9d93f218ee359e61", - "sha256:47f3de503fe7c1245f6f03bea7e8d3ec11c6c4a2ea9ef910e3221c8a15516d62", - "sha256:5e5c933b4c30a988b52e0b7c02641760a5ba046edc5e43d3b94a74c9fc57c1b3", - "sha256:8f62574857ee1de9f770baf04dde4165e30b15ad97ba03ceac65f760ff018ac9", - "sha256:a8b7a98d4ce823303145bf3c1a8bdb0f2f4642a414b196f04ad9853ed0c8f830", - "sha256:b50c949608682b12efb0b2717f53256f03636af5f60ac0c1d900df6213910fd6", - "sha256:d66a769b8d687df9024f2985d5137a337f957a0916cf5464d1513eee96a63ff0", - "sha256:fc381d1dd0516343f1440019cedf08a7405f791cd49eef4ae1ea06520bc1c020", - "sha256:fe599e175cb347efc8ee524bcd4b902d11f7262c0e569ececcb89995c15f0a5e" + "sha256:19b270aeaa0099f16d3ca02628546b8baefe2955bbe23224aaf856134eccf1e4", + "sha256:209ba4cc916bab46f64e56b85b090607a676f66b473e6b762e6f1d9d591eb2e8", + "sha256:25b5d0b42fd000320bd7830b349e3b696435f3b329810427a6bcce6a5492cc5c", + "sha256:7c8daa26095f82482307bc717364e7c13f4f1c99659be82890dcfc215194554d", + "sha256:c053062984e61144385022e53678fbded7aea14ebb3e0305ae3592fb219ccfa4", + "sha256:d4198877797a83cbfe9bffa3803602bbe1625dc30d8a097365dbc762e5790faa", + "sha256:e3c97a1555fd6388f857770ff8b9703083de6bf1f9274a002a332d65fbb56c8c", + "sha256:e7cb0ae90dd83727f0c0718634ed56837bfeeee29a5f82a7514c03ee1364c019", + "sha256:f0700d54bcf45424477e46a9f0944155b46fb0639d69728739c0e47bab83f2b9", + "sha256:f1279ab38ecbfae7e456a108c5c0681e4956d5b1090027c1de0f934dfdb4b35c", + "sha256:f4f118245c4a087776e0a8408be33cf09f6c547442c00395fbfb116fac2f8ac2" ], "index": "pypi", - "version": "==4.25.2" + "version": "==4.25.3" }, "pyarrow": { "hashes": [ - "sha256:059bd8f12a70519e46cd64e1ba40e97eae55e0cbe1695edd95384653d7626b23", - "sha256:06ff1264fe4448e8d02073f5ce45a9f934c0f3db0a04460d0b01ff28befc3696", - "sha256:1e6987c5274fb87d66bb36816afb6f65707546b3c45c44c28e3c4133c010a881", - "sha256:209bac546942b0d8edc8debda248364f7f668e4aad4741bae58e67d40e5fcf75", - "sha256:20e003a23a13da963f43e2b432483fdd8c38dc8882cd145f09f21792e1cf22a1", - "sha256:22a768987a16bb46220cef490c56c671993fbee8fd0475febac0b3e16b00a10e", - "sha256:2cc61593c8e66194c7cdfae594503e91b926a228fba40b5cf25cc593563bcd07", - "sha256:2dbba05e98f247f17e64303eb876f4a80fcd32f73c7e9ad975a83834d81f3fda", - "sha256:32356bfb58b36059773f49e4e214996888eeea3a08893e7dbde44753799b2a02", - "sha256:36cef6ba12b499d864d1def3e990f97949e0b79400d08b7cf74504ffbd3eb025", - "sha256:37c233ddbce0c67a76c0985612fef27c0c92aef9413cf5aa56952f359fcb7379", - "sha256:3c0fa3bfdb0305ffe09810f9d3e2e50a2787e3a07063001dcd7adae0cee3601a", - "sha256:3f16111f9ab27e60b391c5f6d197510e3ad6654e73857b4e394861fc79c37200", - "sha256:52809ee69d4dbf2241c0e4366d949ba035cbcf48409bf404f071f624ed313a2b", - "sha256:5c1da70d668af5620b8ba0a23f229030a4cd6c5f24a616a146f30d2386fec422", - "sha256:63ac901baec9369d6aae1cbe6cca11178fb018a8d45068aaf5bb54f94804a866", - "sha256:64df2bf1ef2ef14cee531e2dfe03dd924017650ffaa6f9513d7a1bb291e59c15", - "sha256:66e986dc859712acb0bd45601229021f3ffcdfc49044b64c6d071aaf4fa49e98", - "sha256:6dd4f4b472ccf4042f1eab77e6c8bce574543f54d2135c7e396f413046397d5a", - "sha256:75ee0efe7a87a687ae303d63037d08a48ef9ea0127064df18267252cfe2e9541", - "sha256:76fc257559404ea5f1306ea9a3ff0541bf996ff3f7b9209fc517b5e83811fa8e", - "sha256:78ea56f62fb7c0ae8ecb9afdd7893e3a7dbeb0b04106f5c08dbb23f9c0157591", - "sha256:87482af32e5a0c0cce2d12eb3c039dd1d853bd905b04f3f953f147c7a196915b", - "sha256:87e879323f256cb04267bb365add7208f302df942eb943c93a9dfeb8f44840b1", - "sha256:a01d0052d2a294a5f56cc1862933014e696aa08cc7b620e8c0cce5a5d362e976", - "sha256:a25eb2421a58e861f6ca91f43339d215476f4fe159eca603c55950c14f378cc5", - "sha256:a51fee3a7db4d37f8cda3ea96f32530620d43b0489d169b285d774da48ca9785", - "sha256:a898d134d00b1eca04998e9d286e19653f9d0fcb99587310cd10270907452a6b", - "sha256:b0c4a18e00f3a32398a7f31da47fefcd7a927545b396e1f15d0c85c2f2c778cd", - "sha256:ba9fe808596c5dbd08b3aeffe901e5f81095baaa28e7d5118e01354c64f22807", - "sha256:c65bf4fd06584f058420238bc47a316e80dda01ec0dfb3044594128a6c2db794", - "sha256:c87824a5ac52be210d32906c715f4ed7053d0180c1060ae3ff9b7e560f53f944", - "sha256:e354fba8490de258be7687f341bc04aba181fc8aa1f71e4584f9890d9cb2dec2", - "sha256:e4b123ad0f6add92de898214d404e488167b87b5dd86e9a434126bc2b7a5578d", - "sha256:f7d029f20ef56673a9730766023459ece397a05001f4e4d13805111d7c2108c0", - "sha256:fc0de7575e841f1595ac07e5bc631084fd06ca8b03c0f2ecece733d23cd5102a" + "sha256:06ebccb6f8cb7357de85f60d5da50e83507954af617d7b05f48af1621d331c9a", + "sha256:0d07de3ee730647a600037bc1d7b7994067ed64d0eba797ac74b2bc77384f4c2", + "sha256:0d27bf89dfc2576f6206e9cd6cf7a107c9c06dc13d53bbc25b0bd4556f19cf5f", + "sha256:0d32000693deff8dc5df444b032b5985a48592c0697cb6e3071a5d59888714e2", + "sha256:15fbb22ea96d11f0b5768504a3f961edab25eaf4197c341720c4a387f6c60315", + "sha256:17e23b9a65a70cc733d8b738baa6ad3722298fa0c81d88f63ff94bf25eaa77b9", + "sha256:185d121b50836379fe012753cf15c4ba9638bda9645183ab36246923875f8d1b", + "sha256:18da9b76a36a954665ccca8aa6bd9f46c1145f79c0bb8f4f244f5f8e799bca55", + "sha256:19741c4dbbbc986d38856ee7ddfdd6a00fc3b0fc2d928795b95410d38bb97d15", + "sha256:25233642583bf658f629eb230b9bb79d9af4d9f9229890b3c878699c82f7d11e", + "sha256:2e51ca1d6ed7f2e9d5c3c83decf27b0d17bb207a7dea986e8dc3e24f80ff7d6f", + "sha256:2e73cfc4a99e796727919c5541c65bb88b973377501e39b9842ea71401ca6c1c", + "sha256:31a1851751433d89a986616015841977e0a188662fcffd1a5677453f1df2de0a", + "sha256:3b20bd67c94b3a2ea0a749d2a5712fc845a69cb5d52e78e6449bbd295611f3aa", + "sha256:4740cc41e2ba5d641071d0ab5e9ef9b5e6e8c7611351a5cb7c1d175eaf43674a", + "sha256:48be160782c0556156d91adbdd5a4a7e719f8d407cb46ae3bb4eaee09b3111bd", + "sha256:8785bb10d5d6fd5e15d718ee1d1f914fe768bf8b4d1e5e9bf253de8a26cb1628", + "sha256:98100e0268d04e0eec47b73f20b39c45b4006f3c4233719c3848aa27a03c1aef", + "sha256:99f7549779b6e434467d2aa43ab2b7224dd9e41bdde486020bae198978c9e05e", + "sha256:9cf389d444b0f41d9fe1444b70650fea31e9d52cfcb5f818b7888b91b586efff", + "sha256:a33a64576fddfbec0a44112eaf844c20853647ca833e9a647bfae0582b2ff94b", + "sha256:a8914cd176f448e09746037b0c6b3a9d7688cef451ec5735094055116857580c", + "sha256:b04707f1979815f5e49824ce52d1dceb46e2f12909a48a6a753fe7cafbc44a0c", + "sha256:b5f5705ab977947a43ac83b52ade3b881eb6e95fcc02d76f501d549a210ba77f", + "sha256:ba8ac20693c0bb0bf4b238751d4409e62852004a8cf031c73b0e0962b03e45e3", + "sha256:bf9251264247ecfe93e5f5a0cd43b8ae834f1e61d1abca22da55b20c788417f6", + "sha256:d0ebea336b535b37eee9eee31761813086d33ed06de9ab6fc6aaa0bace7b250c", + "sha256:ddf5aace92d520d3d2a20031d8b0ec27b4395cab9f74e07cc95edf42a5cc0147", + "sha256:ddfe389a08ea374972bd4065d5f25d14e36b43ebc22fc75f7b951f24378bf0b5", + "sha256:e1369af39587b794873b8a307cc6623a3b1194e69399af0efd05bb202195a5a7", + "sha256:e6b6d3cd35fbb93b70ade1336022cc1147b95ec6af7d36906ca7fe432eb09710", + "sha256:f07fdffe4fd5b15f5ec15c8b64584868d063bc22b86b46c9695624ca3505b7b4", + "sha256:f2c5fb249caa17b94e2b9278b36a05ce03d3180e6da0c4c3b3ce5b2788f30eed", + "sha256:f68f409e7b283c085f2da014f9ef81e885d90dcd733bd648cfba3ef265961848", + "sha256:fbef391b63f708e103df99fbaa3acf9f671d77a183a07546ba2f2c297b361e83", + "sha256:febde33305f1498f6df85e8020bca496d0e9ebf2093bab9e0f65e2b4ae2b3444" ], "index": "pypi", - "version": "==14.0.2" + "version": "==16.1.0" }, "pyasn1": { "hashes": [ - "sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58", - "sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c" + "sha256:3a35ab2c4b5ef98e17dfdec8ab074046fbda76e281c5a706ccd82328cfc8f64c", + "sha256:cca4bb0f2df5504f02f6f8a775b6e416ff9b0b3b16f7ee80b5a3153d9b804473" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==0.5.1" + "markers": "python_version >= '3.8'", + "version": "==0.6.0" }, "pyasn1-modules": { "hashes": [ - "sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c", - "sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d" + "sha256:831dbcea1b177b28c9baddf4c6d1013c24c3accd14a1873fffaa6a2e905f17b6", + "sha256:be04f15b66c206eed667e0bb5ab27e2b1855ea54a842e5037738099e8ca4ae0b" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==0.3.0" + "markers": "python_version >= '3.8'", + "version": "==0.4.0" }, "pyathena": { "hashes": [ - "sha256:70e4e428a78dcc772a4437d96974740ab2fbe52807e75d361fcf6db85e0bf4ce", - "sha256:cf5c1bfd9d4806e3be6851bb0877ba84e03e3a01248c8be58a6cc9aecabc1ab8" + "sha256:2b2c49523bd72630dc57587b804be91bb379a772e5b5c7b195ccf7a64fa17715", + "sha256:b95627871409cba06f499ffb25b2993ed13eb8990eb4466b777b93a247d9ddff" ], "index": "pypi", - "version": "==3.1.0" + "version": "==3.8.2" }, "pycparser": { "hashes": [ - "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9", - "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206" + "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6", + "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc" ], - "version": "==2.21" + "markers": "python_version >= '3.8'", + "version": "==2.22" }, "pygments": { "hashes": [ - "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c", - "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367" + "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199", + "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a" ], - "markers": "python_version >= '3.7'", - "version": "==2.17.2" + "markers": "python_version >= '3.8'", + "version": "==2.18.0" }, "pyjwt": { "hashes": [ @@ -982,42 +980,42 @@ }, "pylint": { "hashes": [ - "sha256:27a8d4c7ddc8c2f8c18aa0050148f89ffc09838142193fdbe98f172781a3ff87", - "sha256:f4fcac7ae74cfe36bc8451e931d8438e4a476c20314b1101c458ad0f05191fad" + "sha256:9f20c05398520474dac03d7abb21ab93181f91d4c110e1e0b32bc0d016c34fa4", + "sha256:ad8baf17c8ea5502f23ae38d7c1b7ec78bd865ce34af9a0b986282e2611a8ff2" ], "index": "pypi", - "version": "==2.17.7" + "version": "==3.2.0" }, "pyopenssl": { "hashes": [ - "sha256:6756834481d9ed5470f4a9393455154bc92fe7a64b7bc6ee2c804e78c52099b2", - "sha256:6b2cba5cc46e822750ec3e5a81ee12819850b11303630d575e98108a079c2b12" + "sha256:17ed5be5936449c5418d1cd269a1a9e9081bc54c17aed272b45856a3d3dc86ad", + "sha256:cabed4bfaa5df9f1a16c0ef64a0cb65318b5cd077a7eda7d6970131ca2f41a6f" ], "markers": "python_version >= '3.7'", - "version": "==23.3.0" + "version": "==24.1.0" }, "python-dateutil": { "hashes": [ - "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", - "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", + "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.8.2" + "version": "==2.9.0.post0" }, "pytz": { "hashes": [ - "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b", - "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7" + "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812", + "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319" ], - "version": "==2023.3.post1" + "version": "==2024.1" }, "readme-renderer": { "hashes": [ - "sha256:13d039515c1f24de668e2c93f2e877b9dbe6c6c32328b90a40a49d8b2b85f36d", - "sha256:2d55489f83be4992fe4454939d1a051c33edbab778e82761d060c9fc6b308cd1" + "sha256:1818dd28140813509eeed8d62687f7cd4f7bad90d4db586001c5dc09d4fde311", + "sha256:19db308d86ecd60e5affa3b2a98f017af384678c63c88e5d4556a380e674f3f9" ], "markers": "python_version >= '3.8'", - "version": "==42.0" + "version": "==43.0" }, "requests": { "hashes": [ @@ -1045,11 +1043,11 @@ }, "rich": { "hashes": [ - "sha256:5cb5123b5cf9ee70584244246816e9114227e0b98ad9176eede6ad54bf5403fa", - "sha256:6da14c108c4866ee9520bbffa71f6fe3962e193b7da68720583850cd4548e235" + "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222", + "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432" ], "markers": "python_full_version >= '3.7.0'", - "version": "==13.7.0" + "version": "==13.7.1" }, "rsa": { "hashes": [ @@ -1061,11 +1059,11 @@ }, "s3transfer": { "hashes": [ - "sha256:3cdb40f5cfa6966e812209d0994f2a4709b561c88e90cf00c2696d2df4e56b2e", - "sha256:d0c8bbf672d5eebbe4e57945e23b972d963f07d82f661cabf678a5c88831595b" + "sha256:5683916b4c724f799e600f41dd9e10a9ff19871bf87623cc8f491cb4f5fa0a19", + "sha256:ceb252b11bcf87080fb7850a224fb6e05c8a776bab8f2b64b7f25b969464839d" ], "markers": "python_version >= '3.8'", - "version": "==0.10.0" + "version": "==0.10.1" }, "secretstorage": { "hashes": [ @@ -1077,11 +1075,11 @@ }, "setuptools": { "hashes": [ - "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05", - "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78" + "sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987", + "sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32" ], "index": "pypi", - "version": "==69.0.3" + "version": "==69.5.1" }, "six": { "hashes": [ @@ -1093,38 +1091,43 @@ }, "snowflake-connector-python": { "hashes": [ - "sha256:1058ab5c98cc62fde8b3f021f0a5076cb7865b5cdab8a9bccde0df88b9e91334", - "sha256:15667a918780d79da755e6a60bbf6918051854951e8f56ccdf5692283e9a8479", - "sha256:1b51fe000c8cf6372d30b73c7136275e52788e6af47010cd1984c9fb03378e86", - "sha256:2b93f55989f80d69278e0f40a7a1c0e737806b7c0ddb0351513a752b837243e8", - "sha256:4093b38cf9abf95c38119f0b23b07e23dc7a8689b956cd5d34975e1875741f20", - "sha256:4916f9b4a0efd7c96d1fa50a157e05907b6935f91492cca7f200b43cc178a25e", - "sha256:4ad42613b87f31441d07a8ea242f4c28ed5eb7b6e05986f9e94a7e44b96d3d1e", - "sha256:50dd954ea5918d3242ded69225b72f701963cd9c043ee7d9ab35dc22211611c8", - "sha256:55a6418cec585b050e6f05404f25e62b075a3bbea587dc1f903de15640565c58", - "sha256:7662e2de25b885abe08ab866cf7c7b026ad1af9faa39c25e2c25015ef807abe3", - "sha256:9dfcf178271e892e64e4092b9e011239a066ce5de848afd2efe3f13197a9f8b3", - "sha256:b5db47d4164d6b7a07c413a46f9edc4a1d687e3df44fd9d5fa89a89aecb94a8e", - "sha256:bcbd3102f807ebbbae52b1b5683d45cd7b3dcb0eaec131233ba6b156e8d70fa4", - "sha256:bf8c1ad5aab5304fefa2a4178061a24c96da45e3e3db9d901621e9953e005402", - "sha256:cf5a964fe01b177063f8c44d14df3a72715580bcd195788ec2822090f37330a5", - "sha256:d1fa102f55ee166cc766aeee3f9333b17b4bede6fb088eee1e1f022df15b6d81", - "sha256:d7a11699689a19916e65794ce58dca72b8a40fe6a7eea06764931ede10b47bcc", - "sha256:d810be5b180c6f47ce9b6f989fe64b9984383e4b77e30b284a83e33f229a3a82", - "sha256:f15024c66db5e87d359216ec733a2974d7562aa38f3f18c8b6e65489839e00d7", - "sha256:f7c76aea92b87f6ecd604e9c934aac8a779f2e20f3be1d990d53bb5b6d87b009", - "sha256:fde1e0727e2f23c2a07b49b30e1bc0f49977f965d08ddfda10015b24a2beeb76" + "sha256:05011286f42c52eb3e5a6db59ee3eaf79f3039f3a19d7ffac6f4ee143779c637", + "sha256:12ff767a1b8c48431549ac28884f8bd9647e63a23f470b05f6ab8d143c4b1475", + "sha256:280a8dcca0249e864419564e38764c08f8841900d9872fec2f2855fda494b29f", + "sha256:4602cb19b204bb03e03d65c6d5328467c9efc0fec53ca56768c3747c8dc8a70f", + "sha256:4e5641c70a12da9804b74f350b8cbbdffdc7aca5069b096755abd2a1fdcf5d1b", + "sha256:569301289ada5b0d72d0bd8432b7ca180220335faa6d9a0f7185f60891db6f2c", + "sha256:67bf570230b0cf818e6766c17245c7355a1f5ea27778e54ab8d09e5bb3536ad9", + "sha256:73e9baa531d5156a03bfe5af462cf6193ec2a01cbb575edf7a2dd3b2a35254c7", + "sha256:7c7438e958753bd1174b73581d77c92b0b47a86c38d8ea0ba1ea23c442eb8e75", + "sha256:7e828bc99240433e6552ac4cc4e37f223ae5c51c7880458ddb281668503c7491", + "sha256:858315a2feff86213b079c6293ad8d850a778044c664686802ead8bb1337e1bc", + "sha256:8e2afca4bca70016519d1a7317c498f1d9c56140bf3e40ea40bddcc95fe827ca", + "sha256:8e441484216ed416a6ed338133e23bd991ac4ba2e46531f4d330f61568c49314", + "sha256:a0d3d06d758455c50b998eabc1fd972a1f67faa5c85ef250fd5986f5a41aab0b", + "sha256:aa1e26f9c571d2c4206da5c978c1b345ffd798d3db1f9ae91985e6243c6bf94b", + "sha256:adf16e1ca9f46d3bdf68e955ffa42075ebdb251e3b13b59003d04e4fea7d579a", + "sha256:bb4aced19053c67513cecc92311fa9d3b507b2277698c8e987d404f6f3a49fb2", + "sha256:bfe013ed97b4dd2e191fd6770a14030d29dd0108817d6ce76b9773250dd2d560", + "sha256:c0917c9f9382d830907e1a18ee1208537b203618700a9c671c2a20167b30f574", + "sha256:c889f9f60f915d657e0a0ad2e6cc52cdcafd9bcbfa95a095aadfd8bcae62b819", + "sha256:d19bde29f89b226eb22af4c83134ecb5c229da1d5e960a01b8f495df78dcdc36", + "sha256:d4c5c2a08b39086a5348502652ad4fdf24871d7ab30fd59f6b7b57249158468c", + "sha256:e03361c4749e4d65bf0d223fdea1c2d7a33af53b74e873929a6085d150aff17e", + "sha256:e52bbc1e2e7bda956525b4229d7f87579f8cabd7d5506b12aa754c4bcdc8c8d7", + "sha256:e8cddd4357e70ab55d7aeeed144cbbeb1ff658b563d7d8d307afc06178a367ec", + "sha256:fb1a04b496bbd3e1e2e926df82b2369887b2eea958f535fb934c240bfbabf6c5" ], "index": "pypi", - "version": "==3.6.0" + "version": "==3.10.0" }, "snowflake-sqlalchemy": { "hashes": [ - "sha256:4f1383402ffc89311974bd810dee22003aef4af0f312a0fdb55778333ad1abf7", - "sha256:df022fb73bc04d68dfb3216ebf7a1bfbd14d22def9c38bbe05275beb258adcd0" + "sha256:0a4aa3f391797b22dd7658892f906191fd1d44870503ae7ca9278cddce6e5b89", + "sha256:79191ec3febfb32bcffecd66f2a7dd561bd571345ea4bccbf41cc1fb5c0682ff" ], "index": "pypi", - "version": "==1.5.1" + "version": "==1.5.3" }, "sortedcontainers": { "hashes": [ @@ -1135,71 +1138,71 @@ }, "sqlalchemy": { "hashes": [ - "sha256:0525c4905b4b52d8ccc3c203c9d7ab2a80329ffa077d4bacf31aefda7604dc65", - "sha256:0535d5b57d014d06ceeaeffd816bb3a6e2dddeb670222570b8c4953e2d2ea678", - "sha256:0892e7ac8bc76da499ad3ee8de8da4d7905a3110b952e2a35a940dab1ffa550e", - "sha256:0d661cff58c91726c601cc0ee626bf167b20cc4d7941c93c5f3ac28dc34ddbea", - "sha256:1980e6eb6c9be49ea8f89889989127daafc43f0b1b6843d71efab1514973cca0", - "sha256:1a09d5bd1a40d76ad90e5570530e082ddc000e1d92de495746f6257dc08f166b", - "sha256:245c67c88e63f1523e9216cad6ba3107dea2d3ee19adc359597a628afcabfbcb", - "sha256:2ad16880ccd971ac8e570550fbdef1385e094b022d6fc85ef3ce7df400dddad3", - "sha256:2be4e6294c53f2ec8ea36486b56390e3bcaa052bf3a9a47005687ccf376745d1", - "sha256:2c55040d8ea65414de7c47f1a23823cd9f3fad0dc93e6b6b728fee81230f817b", - "sha256:352df882088a55293f621328ec33b6ffca936ad7f23013b22520542e1ab6ad1b", - "sha256:3823dda635988e6744d4417e13f2e2b5fe76c4bf29dd67e95f98717e1b094cad", - "sha256:38ef80328e3fee2be0a1abe3fe9445d3a2e52a1282ba342d0dab6edf1fef4707", - "sha256:39b02b645632c5fe46b8dd30755682f629ffbb62ff317ecc14c998c21b2896ff", - "sha256:3b0cd89a7bd03f57ae58263d0f828a072d1b440c8c2949f38f3b446148321171", - "sha256:3ec7a0ed9b32afdf337172678a4a0e6419775ba4e649b66f49415615fa47efbd", - "sha256:3f0ef620ecbab46e81035cf3dedfb412a7da35340500ba470f9ce43a1e6c423b", - "sha256:50e074aea505f4427151c286955ea025f51752fa42f9939749336672e0674c81", - "sha256:55e699466106d09f028ab78d3c2e1f621b5ef2c8694598242259e4515715da7c", - "sha256:5e180fff133d21a800c4f050733d59340f40d42364fcb9d14f6a67764bdc48d2", - "sha256:6cacc0b2dd7d22a918a9642fc89840a5d3cee18a0e1fe41080b1141b23b10916", - "sha256:7af40425ac535cbda129d9915edcaa002afe35d84609fd3b9d6a8c46732e02ee", - "sha256:7d8139ca0b9f93890ab899da678816518af74312bb8cd71fb721436a93a93298", - "sha256:7deeae5071930abb3669b5185abb6c33ddfd2398f87660fafdb9e6a5fb0f3f2f", - "sha256:86a22143a4001f53bf58027b044da1fb10d67b62a785fc1390b5c7f089d9838c", - "sha256:8ca484ca11c65e05639ffe80f20d45e6be81fbec7683d6c9a15cd421e6e8b340", - "sha256:8d1d7d63e5d2f4e92a39ae1e897a5d551720179bb8d1254883e7113d3826d43c", - "sha256:8e702e7489f39375601c7ea5a0bef207256828a2bc5986c65cb15cd0cf097a87", - "sha256:a055ba17f4675aadcda3005df2e28a86feb731fdcc865e1f6b4f209ed1225cba", - "sha256:a33cb3f095e7d776ec76e79d92d83117438b6153510770fcd57b9c96f9ef623d", - "sha256:a61184c7289146c8cff06b6b41807c6994c6d437278e72cf00ff7fe1c7a263d1", - "sha256:af55cc207865d641a57f7044e98b08b09220da3d1b13a46f26487cc2f898a072", - "sha256:b00cf0471888823b7a9f722c6c41eb6985cf34f077edcf62695ac4bed6ec01ee", - "sha256:b03850c290c765b87102959ea53299dc9addf76ca08a06ea98383348ae205c99", - "sha256:b97fd5bb6b7c1a64b7ac0632f7ce389b8ab362e7bd5f60654c2a418496be5d7f", - "sha256:c37bc677690fd33932182b85d37433845de612962ed080c3e4d92f758d1bd894", - "sha256:cecb66492440ae8592797dd705a0cbaa6abe0555f4fa6c5f40b078bd2740fc6b", - "sha256:d0a83afab5e062abffcdcbcc74f9d3ba37b2385294dd0927ad65fc6ebe04e054", - "sha256:d3cf56cc36d42908495760b223ca9c2c0f9f0002b4eddc994b24db5fcb86a9e4", - "sha256:e646b19f47d655261b22df9976e572f588185279970efba3d45c377127d35349", - "sha256:e7908c2025eb18394e32d65dd02d2e37e17d733cdbe7d78231c2b6d7eb20cdb9", - "sha256:e8f2df79a46e130235bc5e1bbef4de0583fb19d481eaa0bffa76e8347ea45ec6", - "sha256:eaeeb2464019765bc4340214fca1143081d49972864773f3f1e95dba5c7edc7d", - "sha256:eb18549b770351b54e1ab5da37d22bc530b8bfe2ee31e22b9ebe650640d2ef12", - "sha256:f2e5b6f5cf7c18df66d082604a1d9c7a2d18f7d1dbe9514a2afaccbb51cc4fc3", - "sha256:f8cafa6f885a0ff5e39efa9325195217bb47d5929ab0051636610d24aef45ade" + "sha256:1296f2cdd6db09b98ceb3c93025f0da4835303b8ac46c15c2136e27ee4d18d94", + "sha256:1e135fff2e84103bc15c07edd8569612ce317d64bdb391f49ce57124a73f45c5", + "sha256:1f8e1c6a6b7f8e9407ad9afc0ea41c1f65225ce505b79bc0342159de9c890782", + "sha256:24bb0f81fbbb13d737b7f76d1821ec0b117ce8cbb8ee5e8641ad2de41aa916d3", + "sha256:29d4247313abb2015f8979137fe65f4eaceead5247d39603cc4b4a610936cd2b", + "sha256:2c286fab42e49db23c46ab02479f328b8bdb837d3e281cae546cc4085c83b680", + "sha256:2f251af4c75a675ea42766880ff430ac33291c8d0057acca79710f9e5a77383d", + "sha256:346ed50cb2c30f5d7a03d888e25744154ceac6f0e6e1ab3bc7b5b77138d37710", + "sha256:3491c85df263a5c2157c594f54a1a9c72265b75d3777e61ee13c556d9e43ffc9", + "sha256:427988398d2902de042093d17f2b9619a5ebc605bf6372f7d70e29bde6736842", + "sha256:427c282dd0deba1f07bcbf499cbcc9fe9a626743f5d4989bfdfd3ed3513003dd", + "sha256:49e3772eb3380ac88d35495843daf3c03f094b713e66c7d017e322144a5c6b7c", + "sha256:4dae6001457d4497736e3bc422165f107ecdd70b0d651fab7f731276e8b9e12d", + "sha256:5b5de6af8852500d01398f5047d62ca3431d1e29a331d0b56c3e14cb03f8094c", + "sha256:5bbce5dd7c7735e01d24f5a60177f3e589078f83c8a29e124a6521b76d825b85", + "sha256:5bed4f8c3b69779de9d99eb03fd9ab67a850d74ab0243d1be9d4080e77b6af12", + "sha256:618827c1a1c243d2540314c6e100aee7af09a709bd005bae971686fab6723554", + "sha256:6ab773f9ad848118df7a9bbabca53e3f1002387cdbb6ee81693db808b82aaab0", + "sha256:6e41cb5cda641f3754568d2ed8962f772a7f2b59403b95c60c89f3e0bd25f15e", + "sha256:7027be7930a90d18a386b25ee8af30514c61f3852c7268899f23fdfbd3107181", + "sha256:763bd97c4ebc74136ecf3526b34808c58945023a59927b416acebcd68d1fc126", + "sha256:7d0dbc56cb6af5088f3658982d3d8c1d6a82691f31f7b0da682c7b98fa914e91", + "sha256:80e63bbdc5217dad3485059bdf6f65a7d43f33c8bde619df5c220edf03d87296", + "sha256:80e7f697bccc56ac6eac9e2df5c98b47de57e7006d2e46e1a3c17c546254f6ef", + "sha256:84e10772cfc333eb08d0b7ef808cd76e4a9a30a725fb62a0495877a57ee41d81", + "sha256:853fcfd1f54224ea7aabcf34b227d2b64a08cbac116ecf376907968b29b8e763", + "sha256:99224d621affbb3c1a4f72b631f8393045f4ce647dd3262f12fe3576918f8bf3", + "sha256:a251146b921725547ea1735b060a11e1be705017b568c9f8067ca61e6ef85f20", + "sha256:a551d5f3dc63f096ed41775ceec72fdf91462bb95abdc179010dc95a93957800", + "sha256:a5d2e08d79f5bf250afb4a61426b41026e448da446b55e4770c2afdc1e200fce", + "sha256:a752bff4796bf22803d052d4841ebc3c55c26fb65551f2c96e90ac7c62be763a", + "sha256:afb1672b57f58c0318ad2cff80b384e816735ffc7e848d8aa51e0b0fc2f4b7bb", + "sha256:bcdfb4b47fe04967669874fb1ce782a006756fdbebe7263f6a000e1db969120e", + "sha256:bdb7b4d889631a3b2a81a3347c4c3f031812eb4adeaa3ee4e6b0d028ad1852b5", + "sha256:c124912fd4e1bb9d1e7dc193ed482a9f812769cb1e69363ab68e01801e859821", + "sha256:c294ae4e6bbd060dd79e2bd5bba8b6274d08ffd65b58d106394cb6abbf35cf45", + "sha256:ca5ce82b11731492204cff8845c5e8ca1a4bd1ade85e3b8fcf86e7601bfc6a39", + "sha256:cb8f9e4c4718f111d7b530c4e6fb4d28f9f110eb82e7961412955b3875b66de0", + "sha256:d2de46f5d5396d5331127cfa71f837cca945f9a2b04f7cb5a01949cf676db7d1", + "sha256:d913f8953e098ca931ad7f58797f91deed26b435ec3756478b75c608aa80d139", + "sha256:de9acf369aaadb71a725b7e83a5ef40ca3de1cf4cdc93fa847df6b12d3cd924b", + "sha256:e93983cc0d2edae253b3f2141b0a3fb07e41c76cd79c2ad743fc27eb79c3f6db", + "sha256:f12aaf94f4d9679ca475975578739e12cc5b461172e04d66f7a3c39dd14ffc64", + "sha256:f68016f9a5713684c1507cc37133c28035f29925c75c0df2f9d0f7571e23720a", + "sha256:f7ea11727feb2861deaa293c7971a4df57ef1c90e42cb53f0da40c3468388000", + "sha256:f98dbb8fcc6d1c03ae8ec735d3c62110949a3b8bc6e215053aa27096857afb45" ], "index": "pypi", - "version": "==1.4.51" + "version": "==1.4.52" }, "sqlalchemy-bigquery": { "hashes": [ - "sha256:549b250ad4c75fe9efaff4ee32e08deb488f1886affbb159d8c149b6b537524f", - "sha256:b1a4c2f5b672ca7bb02e1357d6f3aeabbb19a67e986f2ccd2654fb005705e98e" + "sha256:09a2b99b8591d441eef66d34d13057d0f09423fe259fef98c0502df61419d242", + "sha256:99f868cfdd103b13f921ec1c1b748826b4b1187457dda48040da5ab5ba63c705" ], "index": "pypi", - "version": "==1.9.0" + "version": "==1.11.0" }, "tenacity": { "hashes": [ - "sha256:5398ef0d78e63f40007c1fb4c0bff96e1911394d2fa8d194f77619c05ff6cc8a", - "sha256:ce510e327a630c9e1beaf17d42e6ffacc88185044ad85cf74c0a8887c6a0f88c" + "sha256:3649f6443dbc0d9b01b9d8020a9c4ec7a1ff5f6f3c6c8a036ef371f573fe9185", + "sha256:953d4e6ad24357bceffbc9707bc74349aca9d245f68eb65419cf0c249a1949a2" ], - "markers": "python_version >= '3.7'", - "version": "==8.2.3" + "markers": "python_version >= '3.8'", + "version": "==8.3.0" }, "tomli": { "hashes": [ @@ -1211,43 +1214,43 @@ }, "tomlkit": { "hashes": [ - "sha256:75baf5012d06501f07bee5bf8e801b9f343e7aac5a92581f20f80ce632e6b5a4", - "sha256:b0a645a9156dc7cb5d3a1f0d4bab66db287fcb8e0430bdd4664a095ea16414ba" + "sha256:af914f5a9c59ed9d0762c7b64d3b5d5df007448eb9cd2edc8a46b1eafead172f", + "sha256:eef34fba39834d4d6b73c9ba7f3e4d1c417a4e56f89a7e96e090dd0d24b8fb3c" ], "markers": "python_version >= '3.7'", - "version": "==0.12.3" + "version": "==0.12.5" }, "tqdm": { "hashes": [ - "sha256:d302b3c5b53d47bce91fea46679d9c3c6508cf6332229aa1e7d8653723793386", - "sha256:d88e651f9db8d8551a62556d3cff9e3034274ca5d66e93197cf2490e2dcb69c7" + "sha256:b75ca56b413b030bc3f00af51fd2c1a1a5eac6a0c1cca83cbb37a5c52abce644", + "sha256:e4d936c9de8727928f3be6079590e97d9abfe8d39a590be678eb5919ffc186bb" ], "index": "pypi", - "version": "==4.66.1" + "version": "==4.66.4" }, "twine": { "hashes": [ - "sha256:929bc3c280033347a00f847236564d1c52a3e61b1ac2516c97c48f3ceab756d8", - "sha256:9e102ef5fdd5a20661eb88fad46338806c3bd32cf1db729603fe3697b1bc83c8" + "sha256:89b0cc7d370a4b66421cc6102f269aa910fe0f1861c124f573cf2ddedbc10cf4", + "sha256:a262933de0b484c53408f9edae2e7821c1c45a3314ff2df9bdd343aa7ab8edc0" ], "index": "pypi", - "version": "==4.0.2" + "version": "==5.0.0" }, "types-protobuf": { "hashes": [ - "sha256:024f034f3b5e2bb2bbff55ebc4d591ed0d2280d90faceedcb148b9e714a3f3ee", - "sha256:0612ef3156bd80567460a15ac7c109b313f6022f1fee04b4d922ab2789baab79" + "sha256:e4dc2554d342501d5aebc3c71203868b51118340e105fc190e3a64ca1be43831", + "sha256:e6074178109f97efe9f0b20a035ba61d7c3b03e867eb47d254d2b2ab6a805e36" ], "index": "pypi", - "version": "==4.24.0.20240106" + "version": "==5.26.0.20240422" }, "types-python-dateutil": { "hashes": [ - "sha256:1f8db221c3b98e6ca02ea83a58371b22c374f42ae5bbdf186db9c9a76581459f", - "sha256:efbbdc54590d0f16152fa103c9879c7d4a00e82078f6e2cf01769042165acaa2" + "sha256:5d2f2e240b86905e40944dd787db6da9263f0deabef1076ddaed797351ec0202", + "sha256:6b8cb66d960771ce5ff974e9dd45e38facb81718cc1e208b10b1baccbfdbee3b" ], "index": "pypi", - "version": "==2.8.19.20240106" + "version": "==2.9.0.20240316" }, "types-requests": { "hashes": [ @@ -1266,19 +1269,19 @@ }, "typing-extensions": { "hashes": [ - "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783", - "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd" + "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0", + "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a" ], "markers": "python_version < '3.11'", - "version": "==4.9.0" + "version": "==4.11.0" }, "tzdata": { "hashes": [ - "sha256:aa3ace4329eeacda5b7beb7ea08ece826c28d761cda36e747cfbf97996d39bf3", - "sha256:dd54c94f294765522c77399649b4fefd95522479a664a0cec87f41bebc6148c9" + "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd", + "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252" ], "markers": "python_version >= '2'", - "version": "==2023.4" + "version": "==2024.1" }, "urllib3": { "hashes": [ @@ -1290,95 +1293,19 @@ }, "wheel": { "hashes": [ - "sha256:177f9c9b0d45c47873b619f5b650346d632cdc35fb5e4d25058e09c9e581433d", - "sha256:c45be39f7882c9d34243236f2d63cbd58039e360f85d0913425fbd7ceea617a8" + "sha256:465ef92c69fa5c5da2d1cf8ac40559a8c940886afcef87dcf14b9470862f1d85", + "sha256:55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81" ], "index": "pypi", - "version": "==0.42.0" - }, - "wrapt": { - "hashes": [ - "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc", - "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81", - "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09", - "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e", - "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca", - "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0", - "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb", - "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487", - "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40", - "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c", - "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060", - "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202", - "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41", - "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9", - "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b", - "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664", - "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d", - "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362", - "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00", - "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc", - "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1", - "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267", - "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956", - "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966", - "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1", - "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228", - "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72", - "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d", - "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292", - "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0", - "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0", - "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36", - "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c", - "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5", - "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f", - "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73", - "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b", - "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2", - "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593", - "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39", - "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389", - "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf", - "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf", - "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89", - "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c", - "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c", - "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f", - "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440", - "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465", - "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136", - "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b", - "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8", - "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3", - "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8", - "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6", - "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e", - "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f", - "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c", - "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e", - "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8", - "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2", - "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020", - "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35", - "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d", - "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3", - "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537", - "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809", - "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d", - "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a", - "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4" - ], - "markers": "python_version < '3.11'", - "version": "==1.16.0" + "version": "==0.43.0" }, "zipp": { "hashes": [ - "sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31", - "sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0" + "sha256:206f5a15f2af3dbaee80769fb7dc6f249695e940acca08dfb2a4769fe61e538b", + "sha256:2884ed22e7d8961de1c9a05142eb69a247f120291bc0206a00a7642f09b5b715" ], "markers": "python_version >= '3.8'", - "version": "==3.17.0" + "version": "==3.18.1" } }, "develop": {} diff --git a/exabel_data_sdk/client/api/api_client/grpc/base_grpc_client.py b/exabel_data_sdk/client/api/api_client/grpc/base_grpc_client.py index 7ac5dcd..8b93800 100644 --- a/exabel_data_sdk/client/api/api_client/grpc/base_grpc_client.py +++ b/exabel_data_sdk/client/api/api_client/grpc/base_grpc_client.py @@ -1,3 +1,5 @@ +import json + import grpc from exabel_data_sdk.client.api.api_client.exabel_api_group import ExabelApiGroup @@ -5,6 +7,12 @@ SIXTEEN_MEGABYTES_IN_BYTES = 16 * 1024 * 1024 +# Retry parameters +DEFAULT_MAX_ATTEMPTS = 5 +DEFAULT_INITIAL_BACKOFF_SECONDS = 0.1 +DEFAULT_MAX_BACKOFF_SECONDS = 5 +DEFAULT_BACKOFF_MULTIPLIER = 2 + class BaseGrpcClient: """ @@ -18,7 +26,10 @@ def __init__(self, config: ClientConfig, api_group: ExabelApiGroup): "target": f"{api_group.get_host(config)}:{api_group.get_port(config)}", # When importing time series, we may receive a large amount of precondition failure # violations in the trailing metadata, therefore we increase the maximum metadata size. - "options": (("grpc.max_metadata_size", SIXTEEN_MEGABYTES_IN_BYTES),), + "options": ( + ("grpc.max_metadata_size", SIXTEEN_MEGABYTES_IN_BYTES), + ("grpc.service_config", self._get_service_config()), + ), } if config.api_key == "NO_KEY": # Use an insecure channel. This can be used for local testing. @@ -34,3 +45,38 @@ def __init__(self, config: ClientConfig, api_group: ExabelApiGroup): ) for header in self.config.extra_headers: self.metadata.append(header) + + def _get_service_config( + self, + max_attempts: int = DEFAULT_MAX_ATTEMPTS, + initial_backoff: float = DEFAULT_INITIAL_BACKOFF_SECONDS, + max_backoff: float = DEFAULT_MAX_BACKOFF_SECONDS, + backoff_multiplier: int = DEFAULT_BACKOFF_MULTIPLIER, + ) -> str: + """ + Returns a JSON string to use for grpc.service_config option when creating a grpc channel + """ + return json.dumps( + { + "methodConfig": [ + { + "name": [{}], + "retryPolicy": { + "maxAttempts": max_attempts, + "initialBackoff": f"{initial_backoff}s", + "maxBackoff": f"{max_backoff}s", + "backoffMultiplier": backoff_multiplier, + "retryableStatusCodes": [ + "CANCELLED", + "UNKNOWN", + "DEADLINE_EXCEEDED", + "RESOURCE_EXHAUSTED", + "ABORTED", + "INTERNAL", + "UNAVAILABLE", + ], + }, + } + ] + } + ) diff --git a/exabel_data_sdk/client/api/bulk_import.py b/exabel_data_sdk/client/api/bulk_import.py index 0afc3d0..3bcb0a1 100644 --- a/exabel_data_sdk/client/api/bulk_import.py +++ b/exabel_data_sdk/client/api/bulk_import.py @@ -25,6 +25,7 @@ ) from exabel_data_sdk.util.deprecate_arguments import deprecate_arguments from exabel_data_sdk.util.import_ import get_batches_for_import +from exabel_data_sdk.util.logging_thread_pool_executor import LoggingThreadPoolExecutor logger = logging.getLogger(__name__) @@ -47,15 +48,13 @@ def __post_init__(self) -> None: @overload def get_time_series_result_from_violations( resource: TimeSeries, violations: Mapping[str, Violation] - ) -> ResourceCreationResult[TimeSeries]: - ... + ) -> ResourceCreationResult[TimeSeries]: ... @staticmethod @overload def get_time_series_result_from_violations( resource: pd.Series, violations: Mapping[str, Violation] - ) -> ResourceCreationResult[pd.Series]: - ... + ) -> ResourceCreationResult[pd.Series]: ... @staticmethod def get_time_series_result_from_violations( @@ -262,7 +261,11 @@ def _bulk_import( ) else: - with ThreadPoolExecutor(max_workers=threads) as executor: + with ( + LoggingThreadPoolExecutor(max_workers=threads) + if logger.getEffectiveLevel() == logging.DEBUG + else ThreadPoolExecutor(max_workers=threads) + ) as executor: for resource_batch in resource_batches: if not results.abort: # The generic type hints do not guarantee that TResource refers to the same @@ -270,9 +273,9 @@ def _bulk_import( # following function call. executor.submit( _process, - results, # type: ignore[arg-type] - resource_batch, # type: ignore[arg-type] - import_func, # type: ignore[arg-type] + results, + resource_batch, + import_func, # Python 3.9 added support for the shutdown argument 'cancel_futures'. # We should set this argument to True once we have moved to this python # version. diff --git a/exabel_data_sdk/client/api/bulk_insert.py b/exabel_data_sdk/client/api/bulk_insert.py index fdd957a..2a63773 100644 --- a/exabel_data_sdk/client/api/bulk_insert.py +++ b/exabel_data_sdk/client/api/bulk_insert.py @@ -93,9 +93,9 @@ def _bulk_insert( # following function call. executor.submit( _process, - results, # type: ignore[arg-type] - resource, # type: ignore[arg-type] - insert_func, # type: ignore[arg-type] + results, + resource, + insert_func, # Python 3.9 added support for the shutdown argument 'cancel_futures'. # We should set this argument to True once we have moved to this python # version. diff --git a/exabel_data_sdk/client/api/data_classes/prediction_model_run.py b/exabel_data_sdk/client/api/data_classes/prediction_model_run.py index 6b66bc4..a15f798 100644 --- a/exabel_data_sdk/client/api/data_classes/prediction_model_run.py +++ b/exabel_data_sdk/client/api/data_classes/prediction_model_run.py @@ -82,9 +82,11 @@ def from_proto(model_run: ProtoPredictionModelRun) -> "PredictionModelRun": name=model_run.name, description=model_run.description, configuration=ModelConfiguration(model_run.configuration), - configuration_source=model_run.configuration_source - if model_run.HasField("configuration_source") - else None, + configuration_source=( + model_run.configuration_source + if model_run.HasField("configuration_source") + else None + ), auto_activate=model_run.auto_activate, ) diff --git a/exabel_data_sdk/client/api/data_classes/time_series.py b/exabel_data_sdk/client/api/data_classes/time_series.py index 8effa8a..3a8622d 100644 --- a/exabel_data_sdk/client/api/data_classes/time_series.py +++ b/exabel_data_sdk/client/api/data_classes/time_series.py @@ -22,6 +22,7 @@ class Dimension(Enum): DIMENSION_MASS = ProtoUnit.Dimension.DIMENSION_MASS DIMENSION_LENGTH = ProtoUnit.Dimension.DIMENSION_LENGTH DIMENSION_TIME = ProtoUnit.Dimension.DIMENSION_TIME + DIMENSION_RATIO = ProtoUnit.Dimension.DIMENSION_RATIO @classmethod def from_string(cls, dimension: str) -> Dimension: @@ -32,7 +33,7 @@ def from_string(cls, dimension: str) -> Dimension: except KeyError as e: raise ValueError( f"Unknown dimension: {dimension}. " - + "Supported values are: unknown, currency, mass, length, time." + + "Supported values are: unknown, currency, mass, length, time, and ratio." ) from e diff --git a/exabel_data_sdk/client/api/pageable_resource.py b/exabel_data_sdk/client/api/pageable_resource.py index 9386368..613d6c4 100644 --- a/exabel_data_sdk/client/api/pageable_resource.py +++ b/exabel_data_sdk/client/api/pageable_resource.py @@ -38,8 +38,7 @@ def _get_resource_iterator( resource_count = 0 while True: result = pageable_func(**kwargs, page_token=page_token) - for resource in result.results: - yield resource + yield from result.results page_token = result.next_page_token resource_count += len(result.results) if resource_count >= result.total_size: diff --git a/exabel_data_sdk/client/api/resource_creation_result.py b/exabel_data_sdk/client/api/resource_creation_result.py index 80e9753..f618269 100644 --- a/exabel_data_sdk/client/api/resource_creation_result.py +++ b/exabel_data_sdk/client/api/resource_creation_result.py @@ -15,6 +15,7 @@ from exabel_data_sdk.services.csv_loading_constants import ( DEFAULT_ABORT_THRESHOLD, DEFAULT_BULK_LOAD_CHECKPOINTS, + FAILURE_LOG_LIMIT, ) logger = logging.getLogger(__name__) @@ -90,7 +91,11 @@ def get_printable_error(self) -> str: Return a printable request error message if it is set, otherwise return the string representation of the error """ - return self.error.message if self.error and self.error.message else str(self.error) + return ( + f"{self.error.error_type.name}: {self.error.message}" + if self.error and self.error.message + else str(self.error) + ) class ResourceCreationResults(Generic[ResourceT]): @@ -165,7 +170,9 @@ def get_failures(self) -> Sequence[ResourceCreationResult[ResourceT]]: """Return all the failed results.""" return list(filter(lambda r: r.status == ResourceCreationStatus.FAILED, self.results)) - def extract_retryable_failures(self) -> Sequence[ResourceCreationResult[ResourceT]]: + def extract_retryable_failures( + self, log_summary: bool = True + ) -> Sequence[ResourceCreationResult[ResourceT]]: """ Remove all retryable failures from this result set, and return them. @@ -183,6 +190,14 @@ def extract_retryable_failures(self) -> Sequence[ResourceCreationResult[Resource rest.append(result) self.counter.subtract([result.status for result in failed]) self.results = rest + + if log_summary and failed: + errors = [failure.error for failure in failed if failure.error] + error_types = Counter(error.error_type for error in errors) + + logger.info("The following retryable failures were returned:") + for error_type, count in error_types.items(): + logger.info("%d failures with error type: %s", count, error_type.name) return failed def check_failures(self) -> None: @@ -198,7 +213,7 @@ def check_failures(self) -> None: self.abort_threshold * 100, ) - def print_summary(self) -> None: + def print_summary(self, failure_log_limit: Optional[int] = FAILURE_LOG_LIMIT) -> None: """Prints a human legible summary of the resource creation results to screen.""" if self.counter[ResourceCreationStatus.CREATED]: logger.info("%s new resources created", self.counter[ResourceCreationStatus.CREATED]) @@ -208,13 +223,26 @@ def print_summary(self) -> None: logger.info("%s resources upserted", self.counter[ResourceCreationStatus.UPSERTED]) if self.counter[ResourceCreationStatus.FAILED]: logger.warning("%s resources failed", self.counter[ResourceCreationStatus.FAILED]) - for result in self.results: - if result.status == ResourceCreationStatus.FAILED: + failures = self.get_failures() + for i, failure in enumerate(failures): + if failure_log_limit and i > failure_log_limit: logger.warning( - " %s\n %s", - result.get_printable_resource(), - result.get_printable_error(), + "%d resources failed. Only %d resources shown.", + len(failures), + failure_log_limit, ) + break + logger.warning( + " %s\n %s", + failure.get_printable_resource(), + failure.get_printable_error(), + ) + + errors = [failure.error for failure in failures if failure.error] + error_types = Counter(error.error_type for error in errors) + logger.warning("Summary of the errors for the failed resources:") + for error_type, count in error_types.items(): + logger.warning(" %s: %d", error_type.name, count) def print_status(self) -> None: """ diff --git a/exabel_data_sdk/scripts/check_company_identifiers_in_csv.py b/exabel_data_sdk/scripts/check_company_identifiers_in_csv.py index 2f7b999..79af8f5 100644 --- a/exabel_data_sdk/scripts/check_company_identifiers_in_csv.py +++ b/exabel_data_sdk/scripts/check_company_identifiers_in_csv.py @@ -72,9 +72,9 @@ def _process_entity_resource_names( for identifier, entity in entity_resource_names.mapping.items() ) df = pd.DataFrame(all_rows) - df.loc[ - (df["entity"].duplicated(keep=False) & ~df["entity"].isna()), "warning" - ] = "Multiple identifiers mapping to the same company" + df.loc[(df["entity"].duplicated(keep=False) & ~df["entity"].isna()), "warning"] = ( + "Multiple identifiers mapping to the same company" + ) if not keep_all_identifiers: df = df[df["warning"].notnull()] df = df.fillna("").sort_values(["entity", identifier_type]) diff --git a/exabel_data_sdk/scripts/load_time_series_from_csv.py b/exabel_data_sdk/scripts/load_time_series_from_csv.py index e67e052..d112f3f 100644 --- a/exabel_data_sdk/scripts/load_time_series_from_csv.py +++ b/exabel_data_sdk/scripts/load_time_series_from_csv.py @@ -1,6 +1,7 @@ """ This file is here to keep backwards capability with the old name of the time series import script. """ + import sys from exabel_data_sdk.scripts.load_time_series_from_file import LoadTimeSeriesFromFile diff --git a/exabel_data_sdk/scripts/load_time_series_metadata_from_file.py b/exabel_data_sdk/scripts/load_time_series_metadata_from_file.py index 6b46b72..69eb14b 100644 --- a/exabel_data_sdk/scripts/load_time_series_metadata_from_file.py +++ b/exabel_data_sdk/scripts/load_time_series_metadata_from_file.py @@ -91,16 +91,6 @@ def __init__(self, argv: Sequence[str]): "'known_time'. Take care to maintain correct casing in the file when using this " "option.", ) - self.parser.add_argument( - "--unit-type", - required=False, - type=str, - default=None, - help=( - "The unit type of the time series. " - "One of 'currency', 'time', 'mass', 'length' or 'unknown'." - ), - ) def run_script(self, client: ExabelClient, args: argparse.Namespace) -> None: try: @@ -119,7 +109,6 @@ def run_script(self, client: ExabelClient, args: argparse.Namespace) -> None: batch_size=args.batch_size, skip_validation=args.skip_validation, case_sensitive_signals=args.case_sensitive_signals, - unit_type=args.unit_type, ) except FileLoadingException as e: diff --git a/exabel_data_sdk/scripts/upsert_time_series_unit.py b/exabel_data_sdk/scripts/upsert_time_series_unit.py index b338f9b..4e59557 100644 --- a/exabel_data_sdk/scripts/upsert_time_series_unit.py +++ b/exabel_data_sdk/scripts/upsert_time_series_unit.py @@ -35,7 +35,7 @@ def __init__(self, argv: Sequence[str]): default=None, help=( "The unit type of the time series. " - "One of 'unknown', 'currency', 'mass', 'length', 'time'." + "One of 'unknown', 'currency', 'mass', 'length', 'time', 'ratio'." ), ) self.parser.add_argument( diff --git a/exabel_data_sdk/services/csv_entity_loader.py b/exabel_data_sdk/services/csv_entity_loader.py index 9647e80..6e2f676 100644 --- a/exabel_data_sdk/services/csv_entity_loader.py +++ b/exabel_data_sdk/services/csv_entity_loader.py @@ -51,6 +51,7 @@ def load_entities( abort_threshold: Optional[float] = DEFAULT_ABORT_THRESHOLD, batch_size: Optional[int] = None, return_results: bool = True, + total_rows: Optional[int] = None, # Deprecated arguments name_column: Optional[str] = None, # pylint: disable=unused-argument namespace: Optional[str] = None, # pylint: disable=unused-argument @@ -82,6 +83,9 @@ def load_entities( upload to be aborted; if it is `None`, the upload is never aborted batch_size: the number of entities to upload in each batch; if not specified, the entire file will be read into memory and uploaded in a single batch + return_results: if True, returns a FileLoadingResult with info, else returns an + empty FileLoadingResult + total_rows: the total number of rows to be processed """ if dry_run: logger.info("Running dry-run...") @@ -155,6 +159,13 @@ def load_entities( ) if return_results: combined_result.update(result) + if combined_result.processed_rows is not None and total_rows: + logger.info( + "Rows processed: %d / %d. %.1f %%", + combined_result.processed_rows, + total_rows, + 100 * combined_result.processed_rows / total_rows, + ) return combined_result @@ -229,7 +240,7 @@ def _load_entities( "An error occurred while uploading entities.", failures=result.get_failures(), ) - return FileLoadingResult(result) + return FileLoadingResult(result, processed_rows=len(data_frame)) except BulkInsertFailedError as e: # An error summary has already been printed. if error_on_any_failure: diff --git a/exabel_data_sdk/services/csv_exception.py b/exabel_data_sdk/services/csv_exception.py index 72bf768..82e5ed0 100644 --- a/exabel_data_sdk/services/csv_exception.py +++ b/exabel_data_sdk/services/csv_exception.py @@ -1,4 +1,5 @@ """This file aliases an import for backwards compatibility after an exception object was renamed.""" + # pylint: disable=unused-import from exabel_data_sdk.services.file_loading_exception import ( FileLoadingException as CsvLoadingException, diff --git a/exabel_data_sdk/services/csv_loading_constants.py b/exabel_data_sdk/services/csv_loading_constants.py index fe1d0ac..f93dbbf 100644 --- a/exabel_data_sdk/services/csv_loading_constants.py +++ b/exabel_data_sdk/services/csv_loading_constants.py @@ -3,4 +3,5 @@ DEFAULT_NUMBER_OF_THREADS = 40 DEFAULT_NUMBER_OF_THREADS_FOR_IMPORT = 4 DEFAULT_NUMBER_OF_RETRIES = 5 -MAX_THREADS_FOR_IMPORT = 40 +MAX_THREADS_FOR_IMPORT = 100 +FAILURE_LOG_LIMIT = None # type: ignore[var-annotated] diff --git a/exabel_data_sdk/services/csv_loading_result.py b/exabel_data_sdk/services/csv_loading_result.py index 9eb0832..ec3fd25 100644 --- a/exabel_data_sdk/services/csv_loading_result.py +++ b/exabel_data_sdk/services/csv_loading_result.py @@ -1,3 +1,4 @@ """This file aliases an import for backwards compatibility after the result object was renamed.""" + # pylint: disable=unused-import from exabel_data_sdk.services.file_loading_result import FileLoadingResult as CsvLoadingResult diff --git a/exabel_data_sdk/services/csv_reader.py b/exabel_data_sdk/services/csv_reader.py index 6d70c77..5ac2f05 100644 --- a/exabel_data_sdk/services/csv_reader.py +++ b/exabel_data_sdk/services/csv_reader.py @@ -15,8 +15,7 @@ def read_file( *, keep_default_na: bool, nrows: Optional[int], - ) -> pd.DataFrame: - ... + ) -> pd.DataFrame: ... @overload @staticmethod @@ -26,8 +25,7 @@ def read_file( string_columns: Iterable[Union[str, int]], *, keep_default_na: bool, - ) -> pd.DataFrame: - ... + ) -> pd.DataFrame: ... @overload @staticmethod @@ -38,8 +36,7 @@ def read_file( *, keep_default_na: bool, chunksize: int, - ) -> Iterator[pd.DataFrame]: - ... + ) -> Iterator[pd.DataFrame]: ... @overload @staticmethod @@ -50,8 +47,7 @@ def read_file( *, keep_default_na: bool, chunksize: Optional[int], - ) -> Union[pd.DataFrame, Iterator[pd.DataFrame]]: - ... + ) -> Union[pd.DataFrame, Iterator[pd.DataFrame]]: ... @staticmethod def read_file( diff --git a/exabel_data_sdk/services/csv_relationship_loader.py b/exabel_data_sdk/services/csv_relationship_loader.py index 25efeda..f3453b9 100644 --- a/exabel_data_sdk/services/csv_relationship_loader.py +++ b/exabel_data_sdk/services/csv_relationship_loader.py @@ -305,6 +305,7 @@ def load_relationships( abort_threshold: Optional[float] = DEFAULT_ABORT_THRESHOLD, batch_size: Optional[int] = None, return_results: bool = True, + total_rows: Optional[int] = None, # Deprecated arguments: entity_from_column: Optional[str] = None, # pylint: disable=unused-argument entity_to_column: Optional[str] = None, # pylint: disable=unused-argument @@ -345,6 +346,9 @@ def load_relationships( upload to be aborted; if it is `None`, the upload is never aborted batch_size: the number of relationships to upload in each batch; if not specified, the relationship file will be read into memory and uploaded in a single batch + return_results: if True, returns a FileLoadingResult with info, else returns an + empty FileLoadingResult + total_rows: the total number of rows to be processed """ if dry_run: logger.info("Running dry-run...") @@ -414,6 +418,13 @@ def load_relationships( ) if return_results: combined_result.update(result) + if combined_result.processed_rows is not None and total_rows: + logger.info( + "Rows processed: %d / %d. %.1f %%", + combined_result.processed_rows, + total_rows, + 100 * combined_result.processed_rows / total_rows, + ) return combined_result def _load_relationships( @@ -496,7 +507,9 @@ def _load_relationships( if dry_run: logger.info("Loading %d relationships", len(relationships)) logger.info(relationships) - return FileLoadingResult(warnings=list(map(str, warnings))) + return FileLoadingResult( + warnings=list(map(str, warnings)), processed_rows=len(data_frame) + ) try: result = self._client.relationship_api.bulk_create_relationships( @@ -511,7 +524,9 @@ def _load_relationships( "An error occurred while uploading relationships.", failures=result.get_failures(), ) - return FileLoadingResult(result, warnings=list(map(str, warnings))) + return FileLoadingResult( + result, warnings=list(map(str, warnings)), processed_rows=len(data_frame) + ) except BulkInsertFailedError as e: # An error summary has already been printed. if error_on_any_failure: diff --git a/exabel_data_sdk/services/csv_writer.py b/exabel_data_sdk/services/csv_writer.py index 52baee0..0bea039 100644 --- a/exabel_data_sdk/services/csv_writer.py +++ b/exabel_data_sdk/services/csv_writer.py @@ -4,7 +4,7 @@ import pandas as pd -from exabel_data_sdk.services.file_writer import FileWriter +from exabel_data_sdk.services.file_writer import FileWriter, FileWritingResult logger = logging.getLogger(__name__) @@ -13,15 +13,22 @@ class CsvWriter(FileWriter): """Stores a DataFrame in a CSV file.""" @staticmethod - def write_file(df: Union[pd.DataFrame, Iterable[pd.DataFrame]], filepath: str) -> None: + def write_file( + df: Union[pd.DataFrame, Iterable[pd.DataFrame]], filepath: str + ) -> FileWritingResult: + rows = 0 if isinstance(df, pd.DataFrame): df.to_csv(filepath, index=False) + rows = len(df) else: mode = "w" for chunk in df: header = mode == "w" chunk.to_csv(filepath, header=header, index=False, mode=mode) mode = "a" + rows += len(chunk) if os.path.isfile(filepath): logger.info("Wrote CSV file to: %s", filepath) + + return FileWritingResult(rows) diff --git a/exabel_data_sdk/services/excel_writer.py b/exabel_data_sdk/services/excel_writer.py index d297f56..c3f175e 100644 --- a/exabel_data_sdk/services/excel_writer.py +++ b/exabel_data_sdk/services/excel_writer.py @@ -2,15 +2,17 @@ import pandas as pd -from exabel_data_sdk.services.file_writer import FileWriter +from exabel_data_sdk.services.file_writer import FileWriter, FileWritingResult class ExcelWriter(FileWriter): """Stores a DataFrame in an Excel file.""" @staticmethod - def write_file(df: Union[pd.DataFrame, Iterable[pd.DataFrame]], filepath: str) -> None: + def write_file( + df: Union[pd.DataFrame, Iterable[pd.DataFrame]], filepath: str + ) -> FileWritingResult: if isinstance(df, pd.DataFrame): df.to_excel(filepath, index=False) - return + return FileWritingResult(len(df)) raise NotImplementedError("Writing multiple DataFrames to Excel is not supported.") diff --git a/exabel_data_sdk/services/feather_writer.py b/exabel_data_sdk/services/feather_writer.py index a02b936..71c6467 100644 --- a/exabel_data_sdk/services/feather_writer.py +++ b/exabel_data_sdk/services/feather_writer.py @@ -4,7 +4,7 @@ import pandas as pd -from exabel_data_sdk.services.file_writer import FileWriter +from exabel_data_sdk.services.file_writer import FileWriter, FileWritingResult logger = logging.getLogger(__name__) @@ -13,12 +13,18 @@ class FeatherWriter(FileWriter): """Stores a DataFrame in a Feather file.""" @staticmethod - def write_file(df: Union[pd.DataFrame, Iterable[pd.DataFrame]], filepath: str) -> None: + def write_file( + df: Union[pd.DataFrame, Iterable[pd.DataFrame]], filepath: str + ) -> FileWritingResult: + rows = 0 if isinstance(df, pd.DataFrame): df.to_feather(filepath) - return + rows = len(df) + else: + filepath_stem = Path(filepath).stem + for batch_no, chunk in enumerate(df, 1): + feather_file = f"{filepath_stem}_{batch_no}.feather" + chunk.to_feather(feather_file) + rows += len(chunk) - filepath_stem = Path(filepath).stem - for batch_no, chunk in enumerate(df, 1): - feather_file = f"{filepath_stem}_{batch_no}.feather" - chunk.to_feather(feather_file) + return FileWritingResult(rows) diff --git a/exabel_data_sdk/services/file_loading_result.py b/exabel_data_sdk/services/file_loading_result.py index 4f52637..0fe17a2 100644 --- a/exabel_data_sdk/services/file_loading_result.py +++ b/exabel_data_sdk/services/file_loading_result.py @@ -38,6 +38,7 @@ class FileLoadingResult(Generic[ResourceT]): aborted or it was a dry run warnings: a list of warnings aborted: whether uploading was aborted + processed_rows: the number of rows processed """ def __init__( @@ -46,10 +47,12 @@ def __init__( *, warnings: Optional[Sequence[str]] = None, aborted: bool = False, + processed_rows: Optional[int] = None, ): self.results: Optional[ResourceCreationResults[ResourceT]] = results self.warnings = warnings or [] self.aborted = aborted + self.processed_rows = processed_rows def update(self, other: FileLoadingResult[ResourceT]) -> None: """ @@ -61,6 +64,8 @@ def update(self, other: FileLoadingResult[ResourceT]) -> None: self.results.update(other.results) self.warnings = (*self.warnings, *other.warnings) self.aborted = self.aborted or other.aborted + if other.processed_rows is not None: + self.processed_rows = (self.processed_rows or 0) + other.processed_rows class TimeSeriesFileLoadingResult(FileLoadingResult[pd.Series]): @@ -72,6 +77,7 @@ class TimeSeriesFileLoadingResult(FileLoadingResult[pd.Series]): uploading was aborted or it was a dry run warnings: a list of warnings aborted: whether uploading was aborted + rows: the number of rows processed entity_mapping_result: results of mapping entities in the input file created_data_signals: resource names of data API signals which did not exist from before dry_run_results: resource names of resources which would be created in case of a dry @@ -88,6 +94,7 @@ def __init__( *, warnings: Optional[Sequence[str]] = None, aborted: bool = False, + processed_rows: Optional[int] = None, entity_mapping_result: Optional[EntityMappingResult] = None, created_data_signals: Optional[Sequence[str]] = None, dry_run_results: Optional[Sequence[str]] = None, @@ -95,7 +102,7 @@ def __init__( has_known_time: bool = False, replaced: Optional[Sequence[str]] = None, ): - super().__init__(results, warnings=warnings, aborted=aborted) + super().__init__(results, warnings=warnings, aborted=aborted, processed_rows=processed_rows) if entity_mapping_result is None: raise ValueError("Entity mapping result must be set.") self.entity_mapping_result = entity_mapping_result diff --git a/exabel_data_sdk/services/file_time_series_loader.py b/exabel_data_sdk/services/file_time_series_loader.py index bf39182..f993e90 100644 --- a/exabel_data_sdk/services/file_time_series_loader.py +++ b/exabel_data_sdk/services/file_time_series_loader.py @@ -2,6 +2,7 @@ from typing import List, Mapping, MutableSequence, Optional, Sequence, Tuple, Type from google.protobuf.duration_pb2 import Duration +from google.protobuf.field_mask_pb2 import FieldMask from exabel_data_sdk import ExabelClient from exabel_data_sdk.client.api.bulk_insert import BulkInsertFailedError @@ -78,6 +79,8 @@ def load_time_series( replace_existing_time_series: bool = False, replace_existing_data_points: bool = False, return_results: bool = True, + processed_rows: int = 0, + total_rows: Optional[int] = None, # Deprecated arguments create_tag: Optional[bool] = None, # pylint: disable=unused-argument namespace: Optional[str] = None, # pylint: disable=unused-argument @@ -121,6 +124,8 @@ def load_time_series( replace_existing_data_points: if True, any existing time series data points are replaced return_results: if True, returns a list of TimeSeriesFileLoadingResults or otherwise an empty list. + processed_rows: the number of rows already processed + total_rows: the total number of rows to be processed """ if replace_existing_time_series and replace_existing_data_points: raise ValueError( @@ -165,6 +170,14 @@ def load_time_series( replace_existing_data_points=replace_existing_data_points, replaced_time_series=replaced_time_series, ) + if result.processed_rows is not None and total_rows: + processed_rows = processed_rows + result.processed_rows + logger.info( + "Rows processed: %d / %d. %.1f %%", + processed_rows, + total_rows, + 100 * processed_rows / total_rows, + ) if replace_existing_time_series and result.replaced: replaced_time_series.extend(result.replaced) if return_results: @@ -298,7 +311,9 @@ def _load_time_series( [w.query for w in parsed_file.get_entity_lookup_result().warnings], ) series, invalid_series = parsed_file.get_series( - prefix=self._get_signal_prefix(), skip_validation=skip_validation + prefix=self._get_signal_prefix(), + skip_validation=skip_validation, + replace_existing_time_series=replace_existing_time_series, ) if dry_run: @@ -316,6 +331,7 @@ def _load_time_series( dry_run_results=[str(ts.name) for ts in series], sheet_name=parser.sheet_name(), has_known_time=parsed_file.has_known_time(), + processed_rows=len(parsed_file.data), ) try: replaced_in_this_batch = [] @@ -386,6 +402,7 @@ def _load_time_series( sheet_name=parser.sheet_name(), has_known_time=parsed_file.has_known_time(), replaced=replaced_in_this_batch, + processed_rows=len(parsed_file.data), ) except BulkInsertFailedError as e: # An error summary has already been printed. @@ -417,6 +434,8 @@ def batch_delete_time_series_points( skip_validation: bool = False, case_sensitive_signals: bool = False, return_results: bool = True, + processed_rows: int = 0, + total_rows: Optional[int] = None, ) -> Sequence[FileLoadingResult]: """ Load a CSV file to delete the time series data points represented in it. @@ -438,8 +457,10 @@ def batch_delete_time_series_points( process to be aborted; if it is `None`, the upload is never aborted skip_validation: if True, the time series are not validated before deletion case_sensitive_signals: if True, signals are case sensitive - return_results: if True, returns a list of TimeSeriesFileLoadingResults + return_results: if True, returns a list of FileLoadingResults or otherwise an empty list. + processed_rows: the number of rows already processed + total_rows: the total number of rows to be processed """ if dry_run: logger.info("Running dry-run...") @@ -527,7 +548,21 @@ def batch_delete_time_series_points( series, threads, retries, abort_threshold ) if return_results: - results.append(FileLoadingResult(result, warnings=list(map(str, file_warnings)))) + results.append( + FileLoadingResult( + result, + warnings=list(map(str, file_warnings)), + processed_rows=len(parsed_file.data), + ) + ) + if len(parsed_file.data) is not None and total_rows: + processed_rows += len(parsed_file.data) + logger.info( + "Rows processed: %d / %d. %.1f %%", + processed_rows, + total_rows, + 100 * processed_rows / total_rows, + ) return results def load_time_series_metadata( @@ -550,8 +585,9 @@ def load_time_series_metadata( skip_validation: bool = False, case_sensitive_signals: bool = False, return_results: bool = True, - unit_type: Optional[str] = None, - ) -> Sequence[FileLoadingResult]: + processed_rows: int = 0, + total_rows: Optional[int] = None, + ) -> Sequence[TimeSeriesFileLoadingResult]: """ Load a file and upload the time series metadata to the Exabel Data API @@ -582,9 +618,10 @@ def load_time_series_metadata( entire file will be read into memory and uploaded in a single batch skip_validation: if True, the time series are not validated before uploading case_sensitive_signals: if True, signals are case sensitive - return_results: if True, returns a list of TimeSeriesFileLoadingResults + return_results: if True, returns a list of FileLoadingResults or otherwise an empty list. - unit_type: the unit type of the time series. E.g. "currency", "time", "mass", etc. + processed_rows: the number of rows already processed + total_rows: the total number of rows to be processed """ if dry_run: logger.info("Running dry-run...") @@ -643,10 +680,9 @@ def load_time_series_metadata( [w.query for w in parsed_file.get_entity_lookup_result().warnings], ) - series, invalid_series = parsed_file.get_series( # type: ignore[call-arg] + series, invalid_series = parsed_file.get_series( prefix=self._get_signal_prefix(), skip_validation=skip_validation, - unit_type=unit_type, ) if dry_run: logger.info("Running the script would upsert the following time series:") @@ -688,6 +724,7 @@ def load_time_series_metadata( entity_mapping_result=entity_mapping_result, created_data_signals=missing_signals, sheet_name=parser.sheet_name(), + processed_rows=len(parsed_file.data), ) ) except BulkInsertFailedError as e: @@ -706,6 +743,14 @@ def load_time_series_metadata( has_known_time=parsed_file.has_known_time(), ) ) + if len(parsed_file.data) is not None and total_rows: + processed_rows += len(parsed_file.data) + logger.info( + "Rows processed: %d / %d. %.1f %%", + processed_rows, + total_rows, + 100 * processed_rows / total_rows, + ) return results @@ -776,8 +821,10 @@ def _handle_signals( logger.info("Creating the missing signals.") if not dry_run: for signal in missing_signals: - self._client.signal_api.create_signal( + self._client.signal_api.update_signal( Signal(name=signal, display_name=signal), + update_mask=FieldMask(paths=["name"]), + allow_missing=True, create_library_signal=create_library_signal, ) else: diff --git a/exabel_data_sdk/services/file_time_series_parser.py b/exabel_data_sdk/services/file_time_series_parser.py index 5dfa132..4ad7482 100644 --- a/exabel_data_sdk/services/file_time_series_parser.py +++ b/exabel_data_sdk/services/file_time_series_parser.py @@ -2,7 +2,7 @@ import collections import logging import re -from collections import Counter, defaultdict +from collections import defaultdict from dataclasses import dataclass from pathlib import Path from typing import ( @@ -112,7 +112,7 @@ def from_file( logger.info("Reading input data in batches of %d rows. ", batch_size) return ( TimeSeriesFileParser(filename, separator, None, df) - for df in CsvReader.read_file( # pylint: disable=not-an-iterable + for df in CsvReader.read_file( # pylint: disable=all filename, separator, (0,), @@ -301,12 +301,22 @@ def get_entity_names(self) -> Sequence[str]: """Get the entity resource names.""" @abc.abstractmethod - def _get_series_with_potential_duplicate_data_points(self, prefix: str) -> Sequence[pd.Series]: + def _get_series_with_potential_duplicate_data_points( + self, prefix: str, replace_existing_time_series: bool + ) -> Sequence[pd.Series]: """Get the time series, with potential duplicate data points.""" - def get_series(self, prefix: str, *, skip_validation: bool = False) -> ValidatedTimeSeries: + def get_series( + self, + prefix: str, + *, + replace_existing_time_series: bool = False, + skip_validation: bool = False, + ) -> ValidatedTimeSeries: """Get the time series.""" - series = self._get_series_with_potential_duplicate_data_points(prefix) + series = self._get_series_with_potential_duplicate_data_points( + prefix, replace_existing_time_series + ) failures = [] if not skip_validation: series_without_duplicate_data_points = [ @@ -532,14 +542,16 @@ def validate_numeric(self) -> None: if not is_numeric_dtype(values) and any(~values.apply(_is_float)): raise FileLoadingException("Found at least one non-numeric value in the value column.") - def _get_series_with_potential_duplicate_data_points(self, prefix: str) -> Sequence[pd.Series]: + def _get_series_with_potential_duplicate_data_points( + self, prefix: str, replace_existing_time_series: bool = False + ) -> Sequence[pd.Series]: series = [] for (entity, signal), group in self._data.groupby(["entity", "signal"]): # Do not drop nan values, as this format is the only way to actually delete values # by explicitly importing empty values. ts = group["value"] - if ts.empty: + if ts.empty and not replace_existing_time_series: continue ts.name = f"{entity}/{prefix}{signal}" @@ -572,7 +584,8 @@ def _map_entities( entity_type=entity_type, ) data[entity_column] = lookup_result.names - data.rename(columns={entity_column: "entity"}, inplace=True) + data = data.loc[~data[entity_column].isnull()] + data = data.rename(columns={entity_column: "entity"}) return data, lookup_result @@ -634,14 +647,16 @@ def validate_numeric(self) -> None: ].values _check_non_numeric_error(non_numeric_signals) - def _get_series_with_potential_duplicate_data_points(self, prefix: str) -> Sequence[pd.Series]: + def _get_series_with_potential_duplicate_data_points( + self, prefix: str, replace_existing_time_series: bool = False + ) -> Sequence[pd.Series]: series = [] for entity, entity_group in self._data.groupby("entity"): for signal in self.get_signals(): ts = entity_group[signal] ts.dropna(inplace=True) - if ts.empty: + if ts.empty and not replace_existing_time_series: continue ts.name = f"{entity}/{prefix}{signal}" @@ -820,12 +835,16 @@ def validate_numeric(self) -> None: _check_non_numeric_error(non_numeric_signals) - def _get_series_with_potential_duplicate_data_points(self, prefix: str) -> Sequence[pd.Series]: + def _get_series_with_potential_duplicate_data_points( + self, + prefix: str, + replace_existing_time_series: bool = False, + ) -> Sequence[pd.Series]: series = [] for signal, entity in self._data.columns: ts = self._data[(signal, entity)].dropna() - if ts.empty: + if ts.empty and not replace_existing_time_series: continue ts.name = f"{entity}/{prefix}{signal}" @@ -881,21 +900,32 @@ class MetaDataSignalNamesInRows(SignalNamesInRows): +-----------------------+----------+-----------------+-------------+ | AAPL US | my_sig | USD | million | | MSFT US | my_sig | EUR | | + | AAPL US | my_sig2 | ratio | | + | MSFT US | my_sig2 | percent | | +-----------------------+----------+-----------------+-------------+ - The unit, dimension and description columns are optional. + At least one of the columns currency, unit, or description must be present. + Only one of currency and unit can be set. """ - RESERVED_COLUMNS = {"signal", "unit", "dimension", "description"} + RESERVED_COLUMNS = {"signal", "unit", "description"} UNIT_TYPE_COLUMNS = {"currency"} + UNIT_COLUMNS = UNIT_TYPE_COLUMNS.union({"unit"}) + METADATA_COLUMNS = UNIT_COLUMNS.union({"description"}) VALID_COLUMNS = RESERVED_COLUMNS.union(UNIT_TYPE_COLUMNS) + # Units that will be mapped to unit type ratio + RATIO_UNITS = {"percent", "ratio", "%"} + @classmethod def is_valid(cls, data: pd.DataFrame) -> bool: if "signal" not in data.columns: return False entity_column = None + has_metadata_column = False for column in data.columns: + if column in cls.METADATA_COLUMNS: + has_metadata_column = True if column in cls.VALID_COLUMNS: continue if entity_column is None and _is_valid_entity_column( @@ -906,26 +936,23 @@ def is_valid(cls, data: pd.DataFrame) -> bool: return False if _has_duplicate_columns(data.columns): return False - return True + return has_metadata_column def validate_numeric(self) -> None: return None def get_series( - self, prefix: str, *, skip_validation: bool = False, unit_type: Optional[str] = None + self, + prefix: str, + *, + replace_existing_time_series: bool = False, + skip_validation: bool = False, ) -> ParsedTimeSeriesFile.ValidatedTimeSeries: - series = self._get_series_with_potential_duplicates(prefix, unit_type) + series: Sequence[TimeSeries] = self._get_series_with_potential_duplicates(prefix) failures = [] if not skip_validation: - series_names_counter = Counter([ts.name for ts in series]) - series_with_duplicate_names = [] - series_deduplicated = [] - for ts in series: - if series_names_counter[ts.name] > 1: - series_with_duplicate_names.append(ts) - else: - series_deduplicated.append(ts) - duplicates = self._entity_lookup_result.get_duplicates() + series_deduplicated, series_with_duplicate_names = self._get_deduplicated_series(series) + duplicate_entities = self._entity_lookup_result.get_duplicates() failures.extend( [ ResourceCreationResult( @@ -933,7 +960,9 @@ def get_series( ts, RequestError( ErrorType.INVALID_ARGUMENT, - message=self._format_duplicate_message(str(ts.name), duplicates), + message=self._format_duplicate_message( + str(ts.name), duplicate_entities + ), ), ) for ts in series_with_duplicate_names @@ -942,9 +971,7 @@ def get_series( return self.ValidatedTimeSeries(series_deduplicated, failures) return self.ValidatedTimeSeries(series, failures) - def _get_series_with_potential_duplicates( - self, prefix: str, unit_type: Optional[str] = None - ) -> Sequence[TimeSeries]: + def _get_series_with_potential_duplicates(self, prefix: str) -> Sequence[TimeSeries]: """ Creates a list of time series, one for each combination of entity and signal. The name of the time series is constructed from the entity and signal columns. @@ -955,41 +982,81 @@ def _get_series_with_potential_duplicates( The time series are empty, as this format does not contain any data points. """ series = [] - unit_column = "unit" for _, row in self._data.iterrows(): entity, signal = row["entity"], row["signal"] row = row.dropna() - unit = [] + unit_column = "unit" + unit_type = "unknown" + units = [] + has_unit = False for col in row.index: + if col in self.UNIT_COLUMNS: + if has_unit: + raise FileLoadingException( + f"More than one unit specified for {row['entity']}, {row['signal']}" + ) + has_unit = True if col in self.UNIT_TYPE_COLUMNS: unit_column = unit_type = col - break if row.get(unit_column): - unit.append( + unit = row.get(unit_column) + units.append( Unit( - dimension=Dimension.from_string(unit_type) - if unit_type - else Dimension.DIMENSION_UNKNOWN, - unit=row.get(unit_column), + dimension=( + Dimension.DIMENSION_RATIO + if unit in self.RATIO_UNITS + else Dimension.from_string(unit_type) + ), + unit=unit, ) ) series.append( TimeSeries( series=pd.Series(data=[], name=f"{entity}/{prefix}{signal}", dtype=object), - units=Units( - units=unit, - description=row.get("description"), - ) - if (unit or row.get("description")) - else None, + units=( + Units( + units=units, + description=row.get("description"), + ) + if (units or row.get("description")) + else None + ), ) ) return series + @staticmethod + def _get_deduplicated_series( + series: Sequence[TimeSeries], + ) -> Tuple[Sequence[TimeSeries], Sequence[TimeSeries]]: + """Get the deduplicated series and series with duplicate names""" + series_dict = defaultdict(list) + for ts in series: + series_dict[ts.name].append(ts) + + series_with_duplicate_names = [] + series_deduplicated = [] + for ts_name, ts_list in series_dict.items(): + if len(ts_list) > 1: + if any(ts != ts_list[0] for ts in ts_list): + logger.error( + "Time series %s detected with the following different metadata", ts_name + ) + for ts in ts_list: + logger.error(ts.units) + logger.error("The time series metadata will not be uploaded.") + series_with_duplicate_names.extend(ts_list) + else: + series_deduplicated.append(ts_list[0]) + else: + series_deduplicated.extend(ts_list) + + return series_deduplicated, series_with_duplicate_names + @classmethod def _set_index(cls, data: pd.DataFrame) -> pd.DataFrame: return data diff --git a/exabel_data_sdk/services/file_writer.py b/exabel_data_sdk/services/file_writer.py index d6d3d7c..117fb98 100644 --- a/exabel_data_sdk/services/file_writer.py +++ b/exabel_data_sdk/services/file_writer.py @@ -1,13 +1,28 @@ import abc -from typing import Iterable, Union +from dataclasses import dataclass +from typing import Iterable, Optional, Union import pandas as pd +@dataclass +class FileWritingResult: + """ + Contains summary of result after writing to a file or a set of files. + + Attributes: + rows: Number of rows written to a file or a set of files. + """ + + rows: Optional[int] = None + + class FileWriter(abc.ABC): """Base class for file writers.""" @staticmethod @abc.abstractmethod - def write_file(df: Union[pd.DataFrame, Iterable[pd.DataFrame]], filepath: str) -> None: + def write_file( + df: Union[pd.DataFrame, Iterable[pd.DataFrame]], filepath: str + ) -> FileWritingResult: """Write the DataFrame or iterable of DataFrames to a file.""" diff --git a/exabel_data_sdk/services/sql/sql_reader.py b/exabel_data_sdk/services/sql/sql_reader.py index 3b752d2..d30a139 100644 --- a/exabel_data_sdk/services/sql/sql_reader.py +++ b/exabel_data_sdk/services/sql/sql_reader.py @@ -4,6 +4,7 @@ import pandas as pd +from exabel_data_sdk.services.file_writer import FileWritingResult from exabel_data_sdk.services.file_writer_provider import FileWriterProvider logger = logging.getLogger(__name__) @@ -12,6 +13,7 @@ OutputFile = NewType("OutputFile", str) OutputFilePrefix = NewType("OutputFilePrefix", str) BatchSize = NewType("BatchSize", int) +FileFormat = NewType("FileFormat", str) class SqlReader(abc.ABC): @@ -47,7 +49,7 @@ def read_sql_query_and_write_result( output_file: Optional[OutputFile] = None, *, batch_size: Optional[BatchSize] = None, - ) -> None: + ) -> Optional[FileWritingResult]: """ Execute the given query and write the result to the given output file. If no output file is given, print a sample instead. @@ -59,5 +61,5 @@ def read_sql_query_and_write_result( if not output_file: logger.info("No output file specified. Printing sample.") logger.info(self.get_data_frame(df)) - return - FileWriterProvider.get_file_writer(output_file).write_file(df, output_file) + return None + return FileWriterProvider.get_file_writer(output_file).write_file(df, output_file) diff --git a/exabel_data_sdk/stubs/exabel/api/data/v1/time_series_messages_pb2.py b/exabel_data_sdk/stubs/exabel/api/data/v1/time_series_messages_pb2.py index 774864c..f76b3da 100644 --- a/exabel_data_sdk/stubs/exabel/api/data/v1/time_series_messages_pb2.py +++ b/exabel_data_sdk/stubs/exabel/api/data/v1/time_series_messages_pb2.py @@ -11,7 +11,7 @@ from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2 from google.type import decimal_pb2 as google_dot_type_dot_decimal__pb2 from .....protoc_gen_openapiv2.options import annotations_pb2 as protoc__gen__openapiv2_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n-exabel/api/data/v1/time_series_messages.proto\x12\x12exabel.api.data.v1\x1a exabel/api/time/time_range.proto\x1a\x1fgoogle/api/field_behavior.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/wrappers.proto\x1a\x19google/type/decimal.proto\x1a.protoc_gen_openapiv2/options/annotations.proto"\x84\x02\n\nTimeSeries\x12x\n\x04name\x18\x01 \x01(\tBj\x92AbJL"entityTypes/store/entities/ns.apple_store_fifth_avenue/signals/ns.visitors"\xca>\x11\xfa\x02\x0etimeSeriesName\xe2A\x02\x05\x02\x123\n\x06points\x18\x02 \x03(\x0b2#.exabel.api.data.v1.TimeSeriesPoint\x12\x17\n\tread_only\x18\x03 \x01(\x08B\x04\xe2A\x01\x03\x12.\n\x05units\x18\x04 \x01(\x0b2\x19.exabel.api.data.v1.UnitsB\x04\xe2A\x01\x01"\x9e\x01\n\x0fTimeSeriesPoint\x12.\n\x04time\x18\x01 \x01(\x0b2\x1a.google.protobuf.TimestampB\x04\xe2A\x01\x02\x12+\n\x05value\x18\x02 \x01(\x0b2\x1c.google.protobuf.DoubleValue\x12.\n\nknown_time\x18\x03 \x01(\x0b2\x1a.google.protobuf.Timestamp"v\n\x0eTimeSeriesView\x12.\n\ntime_range\x18\x01 \x01(\x0b2\x1a.exabel.api.time.TimeRange\x124\n\nknown_time\x18\x02 \x01(\x0b2\x1a.google.protobuf.TimestampB\x04\xe2A\x01\x01"\x9f\x01\n\x10DefaultKnownTime\x12\x16\n\x0ccurrent_time\x18\x01 \x01(\x08H\x00\x120\n\nknown_time\x18\x02 \x01(\x0b2\x1a.google.protobuf.TimestampH\x00\x120\n\x0btime_offset\x18\x03 \x01(\x0b2\x19.google.protobuf.DurationH\x00B\x0f\n\rspecification"\x83\x01\n\x05Units\x12.\n\x05units\x18\x01 \x03(\x0b2\x18.exabel.api.data.v1.UnitB\x05\xe2A\x02\x01\x05\x12/\n\nmultiplier\x18\x02 \x01(\x0b2\x14.google.type.DecimalB\x05\xe2A\x02\x01\x05\x12\x19\n\x0bdescription\x18\x03 \x01(\tB\x04\xe2A\x01\x01"\xec\x01\n\x04Unit\x12<\n\tdimension\x18\x01 \x01(\x0e2".exabel.api.data.v1.Unit.DimensionB\x05\xe2A\x02\x01\x05\x12\x13\n\x04unit\x18\x02 \x01(\tB\x05\xe2A\x02\x02\x05\x12\x17\n\x08exponent\x18\x03 \x01(\x11B\x05\xe2A\x02\x01\x05"x\n\tDimension\x12\x15\n\x11DIMENSION_UNKNOWN\x10\x00\x12\x16\n\x12DIMENSION_CURRENCY\x10\x01\x12\x12\n\x0eDIMENSION_MASS\x10\x02\x12\x14\n\x10DIMENSION_LENGTH\x10\x03\x12\x12\n\x0eDIMENSION_TIME\x10\x04BK\n\x16com.exabel.api.data.v1B\x17TimeSeriesMessagesProtoP\x01Z\x16exabel.com/api/data/v1b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n-exabel/api/data/v1/time_series_messages.proto\x12\x12exabel.api.data.v1\x1a exabel/api/time/time_range.proto\x1a\x1fgoogle/api/field_behavior.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/wrappers.proto\x1a\x19google/type/decimal.proto\x1a.protoc_gen_openapiv2/options/annotations.proto"\x84\x02\n\nTimeSeries\x12x\n\x04name\x18\x01 \x01(\tBj\x92AbJL"entityTypes/store/entities/ns.apple_store_fifth_avenue/signals/ns.visitors"\xca>\x11\xfa\x02\x0etimeSeriesName\xe2A\x02\x05\x02\x123\n\x06points\x18\x02 \x03(\x0b2#.exabel.api.data.v1.TimeSeriesPoint\x12\x17\n\tread_only\x18\x03 \x01(\x08B\x04\xe2A\x01\x03\x12.\n\x05units\x18\x04 \x01(\x0b2\x19.exabel.api.data.v1.UnitsB\x04\xe2A\x01\x01"\x9e\x01\n\x0fTimeSeriesPoint\x12.\n\x04time\x18\x01 \x01(\x0b2\x1a.google.protobuf.TimestampB\x04\xe2A\x01\x02\x12+\n\x05value\x18\x02 \x01(\x0b2\x1c.google.protobuf.DoubleValue\x12.\n\nknown_time\x18\x03 \x01(\x0b2\x1a.google.protobuf.Timestamp"v\n\x0eTimeSeriesView\x12.\n\ntime_range\x18\x01 \x01(\x0b2\x1a.exabel.api.time.TimeRange\x124\n\nknown_time\x18\x02 \x01(\x0b2\x1a.google.protobuf.TimestampB\x04\xe2A\x01\x01"\x9f\x01\n\x10DefaultKnownTime\x12\x16\n\x0ccurrent_time\x18\x01 \x01(\x08H\x00\x120\n\nknown_time\x18\x02 \x01(\x0b2\x1a.google.protobuf.TimestampH\x00\x120\n\x0btime_offset\x18\x03 \x01(\x0b2\x19.google.protobuf.DurationH\x00B\x0f\n\rspecification"\x93\x01\n\x05Units\x12.\n\x05units\x18\x01 \x03(\x0b2\x18.exabel.api.data.v1.UnitB\x05\xe2A\x02\x01\x05\x12/\n\nmultiplier\x18\x02 \x01(\x0b2\x14.google.type.DecimalB\x05\xe2A\x02\x01\x05\x12\x19\n\x0bdescription\x18\x03 \x01(\tB\x04\xe2A\x01\x01J\x04\x08\x04\x10\x05R\x08is_ratio"\x82\x02\n\x04Unit\x12<\n\tdimension\x18\x01 \x01(\x0e2".exabel.api.data.v1.Unit.DimensionB\x05\xe2A\x02\x01\x05\x12\x13\n\x04unit\x18\x02 \x01(\tB\x05\xe2A\x02\x01\x05\x12\x17\n\x08exponent\x18\x03 \x01(\x11B\x05\xe2A\x02\x01\x05"\x8d\x01\n\tDimension\x12\x15\n\x11DIMENSION_UNKNOWN\x10\x00\x12\x16\n\x12DIMENSION_CURRENCY\x10\x01\x12\x12\n\x0eDIMENSION_MASS\x10\x02\x12\x14\n\x10DIMENSION_LENGTH\x10\x03\x12\x12\n\x0eDIMENSION_TIME\x10\x04\x12\x13\n\x0fDIMENSION_RATIO\x10\x05BK\n\x16com.exabel.api.data.v1B\x17TimeSeriesMessagesProtoP\x01Z\x16exabel.com/api/data/v1b\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'exabel.api.data.v1.time_series_messages_pb2', _globals) @@ -37,7 +37,7 @@ _globals['_UNIT'].fields_by_name['dimension']._options = None _globals['_UNIT'].fields_by_name['dimension']._serialized_options = b'\xe2A\x02\x01\x05' _globals['_UNIT'].fields_by_name['unit']._options = None - _globals['_UNIT'].fields_by_name['unit']._serialized_options = b'\xe2A\x02\x02\x05' + _globals['_UNIT'].fields_by_name['unit']._serialized_options = b'\xe2A\x02\x01\x05' _globals['_UNIT'].fields_by_name['exponent']._options = None _globals['_UNIT'].fields_by_name['exponent']._serialized_options = b'\xe2A\x02\x01\x05' _globals['_TIMESERIES']._serialized_start = 309 @@ -49,8 +49,8 @@ _globals['_DEFAULTKNOWNTIME']._serialized_start = 853 _globals['_DEFAULTKNOWNTIME']._serialized_end = 1012 _globals['_UNITS']._serialized_start = 1015 - _globals['_UNITS']._serialized_end = 1146 - _globals['_UNIT']._serialized_start = 1149 - _globals['_UNIT']._serialized_end = 1385 - _globals['_UNIT_DIMENSION']._serialized_start = 1265 - _globals['_UNIT_DIMENSION']._serialized_end = 1385 \ No newline at end of file + _globals['_UNITS']._serialized_end = 1162 + _globals['_UNIT']._serialized_start = 1165 + _globals['_UNIT']._serialized_end = 1423 + _globals['_UNIT_DIMENSION']._serialized_start = 1282 + _globals['_UNIT_DIMENSION']._serialized_end = 1423 \ No newline at end of file diff --git a/exabel_data_sdk/stubs/exabel/api/data/v1/time_series_messages_pb2.pyi b/exabel_data_sdk/stubs/exabel/api/data/v1/time_series_messages_pb2.pyi index 8e491a6..f85a572 100644 --- a/exabel_data_sdk/stubs/exabel/api/data/v1/time_series_messages_pb2.pyi +++ b/exabel_data_sdk/stubs/exabel/api/data/v1/time_series_messages_pb2.pyi @@ -177,7 +177,7 @@ class Units(google.protobuf.message.Message): """The product of all individual unit parts of this unit. For instance, if a time series measures speed and is given in meters per second, it would have one unit `{ dimension: DIMENSION_LENGTH, unit: 'm' }` and one unit - `{ dimension: DIMENSION_TIME, unit: 's', exponent: -1 }. And if a time series measures a + `{ dimension: DIMENSION_TIME, unit: 's', exponent: -1 }.` And if a time series measures a monetary amount and is specified in United States dollars, it would have the single unit `{ dimension: DIMENSION_CURRENCY, unit: 'USD' }`. """ @@ -222,6 +222,8 @@ class Unit(google.protobuf.message.Message): 'The dimension is a one dimensional size. The SI unit of length is "m", but other units may\n also be used.\n ' DIMENSION_TIME: Unit._Dimension.ValueType 'The dimension is an amount of time. The SI unit of time is "s", but other units may also be\n used.\n ' + DIMENSION_RATIO: Unit._Dimension.ValueType + 'This unit is a ratio (strictly speaking without a dimension). Symbol is typically\n empty or "%".\n ' class Dimension(_Dimension, metaclass=_DimensionEnumTypeWrapper): """The supported dimensions in the Exabel platform.""" @@ -235,13 +237,15 @@ class Unit(google.protobuf.message.Message): 'The dimension is a one dimensional size. The SI unit of length is "m", but other units may\n also be used.\n ' DIMENSION_TIME: Unit.Dimension.ValueType 'The dimension is an amount of time. The SI unit of time is "s", but other units may also be\n used.\n ' + DIMENSION_RATIO: Unit.Dimension.ValueType + 'This unit is a ratio (strictly speaking without a dimension). Symbol is typically\n empty or "%".\n ' DIMENSION_FIELD_NUMBER: builtins.int UNIT_FIELD_NUMBER: builtins.int EXPONENT_FIELD_NUMBER: builtins.int dimension: global___Unit.Dimension.ValueType 'The dimension of this unit.' unit: builtins.str - 'The short hand symbol of a dimension of this unit, for instance "m" or "EUR".' + 'The short hand symbol of a dimension of this unit, for instance "m" or "EUR".\n Required, except for ratio units.\n ' exponent: builtins.int "The exponent (power) of this unit. It can be positive or negative, but if it is 0, the unit's\n exponent defaults to the value 1.\n " diff --git a/exabel_data_sdk/tests/client/api/mock_signal_api.py b/exabel_data_sdk/tests/client/api/mock_signal_api.py index 7a94c2e..94584f0 100644 --- a/exabel_data_sdk/tests/client/api/mock_signal_api.py +++ b/exabel_data_sdk/tests/client/api/mock_signal_api.py @@ -38,6 +38,8 @@ def update_signal( allow_missing: bool = False, create_library_signal: bool = False, ) -> Signal: + if allow_missing and signal.name not in self.signals.resources and create_library_signal: + self.created_library_signals.append(signal) return self.signals.update(signal, allow_missing=allow_missing) def delete_signal(self, name: str) -> None: diff --git a/exabel_data_sdk/tests/resources/data/timeseries_unit_type_currency.csv b/exabel_data_sdk/tests/resources/data/timeseries_metadata_currency.csv similarity index 100% rename from exabel_data_sdk/tests/resources/data/timeseries_unit_type_currency.csv rename to exabel_data_sdk/tests/resources/data/timeseries_metadata_currency.csv diff --git a/exabel_data_sdk/tests/resources/data/timeseries_no_unit_type.csv b/exabel_data_sdk/tests/resources/data/timeseries_metadata_description.csv similarity index 100% rename from exabel_data_sdk/tests/resources/data/timeseries_no_unit_type.csv rename to exabel_data_sdk/tests/resources/data/timeseries_metadata_description.csv diff --git a/exabel_data_sdk/tests/resources/data/timeseries_unit_type_argument.csv b/exabel_data_sdk/tests/resources/data/timeseries_metadata_dimension_unit.csv similarity index 78% rename from exabel_data_sdk/tests/resources/data/timeseries_unit_type_argument.csv rename to exabel_data_sdk/tests/resources/data/timeseries_metadata_dimension_unit.csv index 39b9d99..50c5aa5 100644 --- a/exabel_data_sdk/tests/resources/data/timeseries_unit_type_argument.csv +++ b/exabel_data_sdk/tests/resources/data/timeseries_metadata_dimension_unit.csv @@ -2,3 +2,4 @@ entity;signal;unit entityTypes/company/entities/company_A;signal1;percent entityTypes/company/entities/company_A;signal2;bps entityTypes/company/entities/company_B;signal1;ratio +entityTypes/company/entities/company_B;signal2;% diff --git a/exabel_data_sdk/tests/resources/data/timeseries_metadata_invalid.csv b/exabel_data_sdk/tests/resources/data/timeseries_metadata_invalid.csv new file mode 100644 index 0000000..9c4a279 --- /dev/null +++ b/exabel_data_sdk/tests/resources/data/timeseries_metadata_invalid.csv @@ -0,0 +1,4 @@ +entity;signal +entityTypes/company/entities/company_A;signal1 +entityTypes/company/entities/company_A;signal2 +entityTypes/company/entities/company_B;signal1 diff --git a/exabel_data_sdk/tests/resources/data/timeseries_metadata_invalid2.csv b/exabel_data_sdk/tests/resources/data/timeseries_metadata_invalid2.csv new file mode 100644 index 0000000..5ab23db --- /dev/null +++ b/exabel_data_sdk/tests/resources/data/timeseries_metadata_invalid2.csv @@ -0,0 +1,2 @@ +entity;signal;unit;currency +entityTypes/company/entities/company_A;signal1;ratio;USD diff --git a/exabel_data_sdk/tests/resources/data/timeseries_metadata_unit.csv b/exabel_data_sdk/tests/resources/data/timeseries_metadata_unit.csv new file mode 100644 index 0000000..9be6573 --- /dev/null +++ b/exabel_data_sdk/tests/resources/data/timeseries_metadata_unit.csv @@ -0,0 +1,6 @@ +entity;signal;unit +entityTypes/company/entities/company_A;signal1;percent +entityTypes/company/entities/company_A;signal2;bps +entityTypes/company/entities/company_A;signal3; +entityTypes/company/entities/company_B;signal1;ratio +entityTypes/company/entities/company_B;signal2;% diff --git a/exabel_data_sdk/tests/resources/data/timeseries_metadata_unit_and_currency.csv b/exabel_data_sdk/tests/resources/data/timeseries_metadata_unit_and_currency.csv new file mode 100644 index 0000000..95e7e0e --- /dev/null +++ b/exabel_data_sdk/tests/resources/data/timeseries_metadata_unit_and_currency.csv @@ -0,0 +1,4 @@ +entity;signal;unit;currency +entityTypes/company/entities/company_A;signal1;;NOK +entityTypes/company/entities/company_B;signal1;;USD +entityTypes/company/entities/company_A;signal2;percent; diff --git a/exabel_data_sdk/tests/scripts/sql/test_sql_script.py b/exabel_data_sdk/tests/scripts/sql/test_sql_script.py index a27aecd..91e2a79 100644 --- a/exabel_data_sdk/tests/scripts/sql/test_sql_script.py +++ b/exabel_data_sdk/tests/scripts/sql/test_sql_script.py @@ -21,17 +21,16 @@ def setUp(self) -> None: @mock.patch("exabel_data_sdk.scripts.sql.sql_script.SQLAlchemyReader") def test_sql_script_without_output_file(self, mock_reader): - mock_config_class = mock.create_autospec(SqlReaderConfiguration) - mock_config_instance = mock_config_class() + mock_config_instance = mock.create_autospec(SqlReaderConfiguration) mock_config_instance.get_connection_string_and_kwargs.return_value = ( "connection-string", "kwargs", ) - mock_config_class.from_args.return_value = mock_config_instance + mock_config_instance.from_args.return_value = mock_config_instance mock_reader.return_value.read_sql_query_and_write_result.return_value = None - script = self.SqlScriptImpl(self.common_args, "sql_script", mock_config_class) + script = self.SqlScriptImpl(self.common_args, "sql_script", mock_config_instance) script.run() - mock_config_class.from_args.assert_called_once_with( + mock_config_instance.from_args.assert_called_once_with( argparse.Namespace( query="SELECT 1 AS A", output_file=None, @@ -45,19 +44,18 @@ def test_sql_script_without_output_file(self, mock_reader): @mock.patch("exabel_data_sdk.scripts.sql.sql_script.SQLAlchemyReader") def test_sql_script_with_output_file(self, mock_reader): - mock_config_class = mock.create_autospec(SqlReaderConfiguration) - mock_config_instance = mock_config_class() + mock_config_instance = mock.create_autospec(SqlReaderConfiguration) mock_config_instance.get_connection_string_and_kwargs.return_value = ( "connection-string", "kwargs", ) - mock_config_class.from_args.return_value = mock_config_instance + mock_config_instance.from_args.return_value = mock_config_instance mock_reader.return_value.read_sql_query_and_write_result.return_value = None script = self.SqlScriptImpl( - self.common_args + ["--output-file", "output_file"], "sql_script", mock_config_class + self.common_args + ["--output-file", "output_file"], "sql_script", mock_config_instance ) script.run() - mock_config_class.from_args.assert_called_once_with( + mock_config_instance.from_args.assert_called_once_with( argparse.Namespace( query="SELECT 1 AS A", output_file="output_file", diff --git a/exabel_data_sdk/tests/scripts/test_load_time_series_from_csv.py b/exabel_data_sdk/tests/scripts/test_load_time_series_from_csv.py index 0bdd5e0..a70ad60 100644 --- a/exabel_data_sdk/tests/scripts/test_load_time_series_from_csv.py +++ b/exabel_data_sdk/tests/scripts/test_load_time_series_from_csv.py @@ -509,7 +509,7 @@ def test_valid_no_create_library_signal(self): self.client.namespace = "acme" script.run_script(self.client, script.parse_arguments()) - call_args_list = self.client.signal_api.create_signal.call_args_list + call_args_list = self.client.signal_api.update_signal.call_args_list create_library_signal_status = call_args_list[0][1]["create_library_signal"] self.assertFalse(create_library_signal_status) @@ -523,7 +523,7 @@ def test_valid_create_library_signal(self): self.client.namespace = "acme" script.run_script(self.client, script.parse_arguments()) - call_args_list = self.client.signal_api.create_signal.call_args_list + call_args_list = self.client.signal_api.update_signal.call_args_list create_library_signal_status = call_args_list[0][1]["create_library_signal"] self.assertTrue(create_library_signal_status) @@ -539,8 +539,8 @@ def test_valid_replace_existing_time_series(self): script.run_script(self.client, script.parse_arguments()) call_args_list = self.client.time_series_api.bulk_upsert_time_series.call_args_list - replace_existing_tine_series_status = call_args_list[0][1]["replace_existing_time_series"] - self.assertTrue(replace_existing_tine_series_status) + replace_existing_time_series_status = call_args_list[0][1]["replace_existing_time_series"] + self.assertTrue(replace_existing_time_series_status) def test_replace_existing_time_series_across_batches(self): args = common_args + [ @@ -663,9 +663,9 @@ def test_load_time_series_with_uppercase_signals_not_existing(self): self.assertEqual(1, len(call_args_list)) series = call_args_list[0][0][0] self.assertEqual(1, len(series)) - call_args_list_create_signal = self.client.signal_api.create_signal.call_args_list - self.assertEqual(1, len(call_args_list_create_signal)) - signal = call_args_list_create_signal[0][0][0] + call_args_list_update_signal = self.client.signal_api.update_signal.call_args_list + self.assertEqual(1, len(call_args_list_update_signal)) + signal = call_args_list_update_signal[0][0][0] self.assertEqual("signals/ns.signal1", signal.name) pd.testing.assert_series_equal( @@ -698,7 +698,7 @@ def test_load_time_series_with_uppercase_signals_existing(self): call_args_list = self.client.time_series_api.bulk_upsert_time_series.call_args_list self.assertEqual(1, len(call_args_list)) - self.assertEqual(0, len(self.client.signal_api.create_signal.call_args_list)) + self.assertEqual(0, len(self.client.signal_api.update_signal.call_args_list)) series = call_args_list[0][0][0] self.assertEqual(1, len(series)) @@ -733,7 +733,7 @@ def test_load_time_series_with_uppercase_signals_existing_and_uppercase_entity_t call_args_list = self.client.time_series_api.bulk_upsert_time_series.call_args_list self.assertEqual(1, len(call_args_list)) - self.assertEqual(0, len(self.client.signal_api.create_signal.call_args_list)) + self.assertEqual(0, len(self.client.signal_api.update_signal.call_args_list)) series = call_args_list[0][0][0] self.assertEqual(1, len(series)) @@ -772,9 +772,9 @@ def test_load_time_series_with_uppercase_signals_not_existing_case_sensitive(sel self.assertEqual(1, len(call_args_list)) series = call_args_list[0][0][0] self.assertEqual(1, len(series)) - call_args_list_create_signal = self.client.signal_api.create_signal.call_args_list - self.assertEqual(1, len(call_args_list_create_signal)) - signal = call_args_list_create_signal[0][0][0] + call_args_list_update_signal = self.client.signal_api.update_signal.call_args_list + self.assertEqual(1, len(call_args_list_update_signal)) + signal = call_args_list_update_signal[0][0][0] self.assertEqual("signals/ns.Signal1", signal.name) pd.testing.assert_series_equal( @@ -813,9 +813,9 @@ def test_load_time_series_with_uppercase_signals_and_lower_case_existing_case_se self.assertEqual(1, len(call_args_list)) series = call_args_list[0][0][0] self.assertEqual(1, len(series)) - call_args_list_create_signal = self.client.signal_api.create_signal.call_args_list - self.assertEqual(1, len(call_args_list_create_signal)) - signal = call_args_list_create_signal[0][0][0] + call_args_list_update_signal = self.client.signal_api.update_signal.call_args_list + self.assertEqual(1, len(call_args_list_update_signal)) + signal = call_args_list_update_signal[0][0][0] self.assertEqual("signals/ns.Signal1", signal.name) pd.testing.assert_series_equal( @@ -847,7 +847,7 @@ def test_load_time_series_with_uppercase_signals_existing_case_sensitive(self): call_args_list = self.client.time_series_api.bulk_upsert_time_series.call_args_list self.assertEqual(1, len(call_args_list)) - self.assertEqual(0, len(self.client.signal_api.create_signal.call_args_list)) + self.assertEqual(0, len(self.client.signal_api.update_signal.call_args_list)) series = call_args_list[0][0][0] self.assertEqual(1, len(series)) @@ -883,7 +883,7 @@ def test_load_time_series_with_mixedcase_signals_existing_and_entity_type_nonexi call_args_list = self.client.time_series_api.bulk_upsert_time_series.call_args_list self.assertEqual(1, len(call_args_list)) - self.assertEqual(0, len(self.client.signal_api.create_signal.call_args_list)) + self.assertEqual(0, len(self.client.signal_api.update_signal.call_args_list)) series = call_args_list[0][0][0] self.assertEqual(1, len(series)) diff --git a/exabel_data_sdk/tests/scripts/test_load_time_series_metadata_from_csv.py b/exabel_data_sdk/tests/scripts/test_load_time_series_metadata_from_csv.py index dfe5017..c5f7268 100644 --- a/exabel_data_sdk/tests/scripts/test_load_time_series_metadata_from_csv.py +++ b/exabel_data_sdk/tests/scripts/test_load_time_series_metadata_from_csv.py @@ -19,7 +19,7 @@ common_args = ["script-name", "--sep", ";", "--api-key", "123"] -class TestUploadTimeSeries(unittest.TestCase): +class TestUploadTimeSeriesMetadata(unittest.TestCase): def setUp(self) -> None: self.client = mock.create_autospec(ExabelClient) self.client.entity_api = mock.create_autospec(EntityApi) @@ -32,15 +32,16 @@ def setUp(self) -> None: def test_get_series(self): data = [ + ["a", "signal1", "USD"], ["a", "signal1", "USD"], ["a", "signal2", "EUR"], ["b", "signal1", "NOK"], ["b", "signal1", np.nan], ] - ts_data = pd.DataFrame(data, columns=["entity", "signal", "unit"]) + ts_data = pd.DataFrame(data, columns=["entity", "signal", "currency"]) - parser = MetaDataSignalNamesInRows.from_data_frame(ts_data, self.client.entity_api, "") - time_series = parser.get_series(prefix="signals/acme.", unit_type="currency") + parsed_file = MetaDataSignalNamesInRows.from_data_frame(ts_data, self.client.entity_api, "") + time_series = parsed_file.get_series(prefix="signals/acme.") self.assertEqual(2, len(time_series.failures)) self.assertEqual(2, len(time_series.valid_series)) @@ -81,9 +82,9 @@ def test_get_series__skip_validation(self): ["b", "signal1", np.nan], ] - ts_data = pd.DataFrame(data, columns=["entity", "signal", "unit"]) - parser = MetaDataSignalNamesInRows.from_data_frame(ts_data, self.client.entity_api, "") - time_series = parser.get_series("signals/acme.", skip_validation=True, unit_type="currency") + ts_data = pd.DataFrame(data, columns=["entity", "signal", "currency"]) + parsed_file = MetaDataSignalNamesInRows.from_data_frame(ts_data, self.client.entity_api, "") + time_series = parsed_file.get_series("signals/acme.", skip_validation=True) self.assertSequenceEqual([], time_series.failures) self.assertEqual(4, len(time_series.valid_series)) @@ -92,10 +93,10 @@ def test_get_series__skip_validation(self): time_series.valid_series[3], ) - def test_read_file__no_unit_type_argument(self): + def test_read_file__description(self): args = common_args + [ "--filename", - "./exabel_data_sdk/tests/resources/data/timeseries_no_unit_type.csv", + "./exabel_data_sdk/tests/resources/data/timeseries_metadata_description.csv", ] script = LoadTimeSeriesMetaDataFromFile(args) script.run_script(self.client, script.parse_arguments()) @@ -127,10 +128,10 @@ def test_read_file__no_unit_type_argument(self): series_by_name["entityTypes/company/entities/company_A/signals/ns.signal2"], ) - def test_read_file__unit_type_currency_column(self): + def test_read_file__currency(self): args = common_args + [ "--filename", - "./exabel_data_sdk/tests/resources/data/timeseries_unit_type_currency.csv", + "./exabel_data_sdk/tests/resources/data/timeseries_metadata_currency.csv", ] script = LoadTimeSeriesMetaDataFromFile(args) script.run_script(self.client, script.parse_arguments()) @@ -177,12 +178,10 @@ def test_read_file__unit_type_currency_column(self): series_by_name["entityTypes/company/entities/company_B/signals/ns.signal1"], ) - def test_read_file__unit_type_argument(self): + def test_read_file__unit(self): args = common_args + [ "--filename", - "./exabel_data_sdk/tests/resources/data/timeseries_unit_type_argument.csv", - "--unit-type", - "unknown", + "./exabel_data_sdk/tests/resources/data/timeseries_metadata_unit.csv", ] script = LoadTimeSeriesMetaDataFromFile(args) script.run_script(self.client, script.parse_arguments()) @@ -190,7 +189,7 @@ def test_read_file__unit_type_argument(self): call_args_list = self.client.time_series_api.bulk_upsert_time_series.call_args_list self.assertEqual(1, len(call_args_list)) series = call_args_list[0][0][0] - self.assertEqual(3, len(series)) + self.assertEqual(5, len(series)) series_by_name = {s.name: s for s in series} ts = TimeSeries( @@ -198,7 +197,7 @@ def test_read_file__unit_type_argument(self): [], name="entityTypes/company/entities/company_A/signals/ns.signal1", dtype=object ), units=Units( - units=[Unit(dimension=Dimension.DIMENSION_UNKNOWN, unit="percent")], + units=[Unit(dimension=Dimension.DIMENSION_RATIO, unit="percent")], ), ) self.assertEqual( @@ -222,19 +221,112 @@ def test_read_file__unit_type_argument(self): [], name="entityTypes/company/entities/company_B/signals/ns.signal1", dtype=object ), units=Units( - units=[Unit(dimension=Dimension.DIMENSION_UNKNOWN, unit="ratio")], + units=[Unit(dimension=Dimension.DIMENSION_RATIO, unit="ratio")], ), ) self.assertEqual( ts, series_by_name["entityTypes/company/entities/company_B/signals/ns.signal1"], ) + ts = TimeSeries( + series=pd.Series( + [], name="entityTypes/company/entities/company_B/signals/ns.signal2", dtype=object + ), + units=Units( + units=[Unit(dimension=Dimension.DIMENSION_RATIO, unit="%")], + ), + ) + self.assertEqual( + ts, + series_by_name["entityTypes/company/entities/company_B/signals/ns.signal2"], + ) + ts = TimeSeries( + series=pd.Series( + [], name="entityTypes/company/entities/company_A/signals/ns.signal3", dtype=object + ), + ) + self.assertEqual( + ts, + series_by_name["entityTypes/company/entities/company_A/signals/ns.signal3"], + ) + + def test_read_file__unit_and_currency(self): + args = common_args + [ + "--filename", + "./exabel_data_sdk/tests/resources/data/timeseries_metadata_unit_and_currency.csv", + ] + script = LoadTimeSeriesMetaDataFromFile(args) + script.run_script(self.client, script.parse_arguments()) + + call_args_list = self.client.time_series_api.bulk_upsert_time_series.call_args_list + self.assertEqual(1, len(call_args_list)) + series = call_args_list[0][0][0] + self.assertEqual(3, len(series)) + series_by_name = {s.name: s for s in series} + + ts = TimeSeries( + series=pd.Series( + [], name="entityTypes/company/entities/company_A/signals/ns.signal1", dtype=object + ), + units=Units( + units=[Unit(dimension=Dimension.DIMENSION_CURRENCY, unit="NOK")], + ), + ) + self.assertEqual( + ts, + series_by_name["entityTypes/company/entities/company_A/signals/ns.signal1"], + ) + ts = TimeSeries( + series=pd.Series( + [], name="entityTypes/company/entities/company_B/signals/ns.signal1", dtype=object + ), + units=Units( + units=[Unit(dimension=Dimension.DIMENSION_CURRENCY, unit="USD")], + ), + ) + self.assertEqual( + ts, + series_by_name["entityTypes/company/entities/company_B/signals/ns.signal1"], + ) + ts = TimeSeries( + series=pd.Series( + [], name="entityTypes/company/entities/company_A/signals/ns.signal2", dtype=object + ), + units=Units( + units=[Unit(dimension=Dimension.DIMENSION_RATIO, unit="percent")], + ), + ) + self.assertEqual( + ts, + series_by_name["entityTypes/company/entities/company_A/signals/ns.signal2"], + ) + + def test_read_file__invalid(self): + args = common_args + [ + "--filename", + "./exabel_data_sdk/tests/resources/data/timeseries_metadata_invalid.csv", + ] + script = LoadTimeSeriesMetaDataFromFile(args) + + with self.assertRaises(SystemExit): + script.run_script(self.client, script.parse_arguments()) + + def test_read_file__invalid2(self): + args = common_args + [ + "--filename", + "./exabel_data_sdk/tests/resources/data/timeseries_metadata_invalid2.csv", + ] + script = LoadTimeSeriesMetaDataFromFile(args) + + with self.assertRaises(SystemExit): + script.run_script(self.client, script.parse_arguments()) def _list_signal(self): return iter( [ Signal("signals/ns.signal1", "The Signal", "A description of the signal"), Signal("signals/ns.signal2", "The Other Signal", "A description of the signal"), + Signal("signals/ns.signal3", "Yet Another Signal", "A description of the signal"), ] ) diff --git a/exabel_data_sdk/tests/services/sql/test_sqlalchemy_reader.py b/exabel_data_sdk/tests/services/sql/test_sqlalchemy_reader.py index f660b3d..0c85103 100644 --- a/exabel_data_sdk/tests/services/sql/test_sqlalchemy_reader.py +++ b/exabel_data_sdk/tests/services/sql/test_sqlalchemy_reader.py @@ -4,7 +4,7 @@ import pandas as pd import pandas.testing as pdt -from exabel_data_sdk.services.file_writer import FileWriter +from exabel_data_sdk.services.file_writer import FileWriter, FileWritingResult from exabel_data_sdk.services.sql.sql_reader_configuration import ConnectionString from exabel_data_sdk.services.sql.sqlalchemy_reader import SQLAlchemyReader from exabel_data_sdk.tests.decorators import requires_modules @@ -18,7 +18,7 @@ class TestSQLAlchemyReader(unittest.TestCase): class _MockFileWriter(FileWriter): @staticmethod - def write_file(df: pd.DataFrame, filepath: str) -> None: + def write_file(df: pd.DataFrame, filepath: str) -> FileWritingResult: raise NotImplementedError() def setUp(self) -> None: diff --git a/exabel_data_sdk/tests/services/test_file_time_series_loader.py b/exabel_data_sdk/tests/services/test_file_time_series_loader.py index e76c26d..0bec6b7 100644 --- a/exabel_data_sdk/tests/services/test_file_time_series_loader.py +++ b/exabel_data_sdk/tests/services/test_file_time_series_loader.py @@ -9,6 +9,7 @@ from exabel_data_sdk.client.api.data_classes.entity_type import EntityType from exabel_data_sdk.client.api.data_classes.signal import Signal from exabel_data_sdk.services.file_loading_exception import FileLoadingException +from exabel_data_sdk.services.file_loading_result import FileLoadingResult from exabel_data_sdk.services.file_time_series_loader import FileTimeSeriesLoader from exabel_data_sdk.stubs.exabel.api.data.v1.all_pb2 import SearchEntitiesResponse, SearchTerm from exabel_data_sdk.tests.client.exabel_mock_client import ExabelMockClient @@ -208,7 +209,9 @@ def test_batch_size(self, mock_from_file): mock_from_file.return_value = (mock_parser for _ in range(no_batches)) client = mock.create_autospec(ExabelClient) loader = FileTimeSeriesLoader(client) - with mock.patch.object(loader, "_load_time_series", return_value="result") as mock_load: + with mock.patch.object( + loader, "_load_time_series", return_value=FileLoadingResult() + ) as mock_load: results = loader.load_time_series( filename="filename", batch_size=batch_size, diff --git a/exabel_data_sdk/tests/services/test_file_time_series_parser.py b/exabel_data_sdk/tests/services/test_file_time_series_parser.py index e0f4416..0fe9edd 100644 --- a/exabel_data_sdk/tests/services/test_file_time_series_parser.py +++ b/exabel_data_sdk/tests/services/test_file_time_series_parser.py @@ -1,22 +1,39 @@ import math import unittest from itertools import zip_longest +from typing import Sequence from unittest import mock import pandas as pd import pandas.testing as pdt +from dateutil import tz +from exabel_data_sdk.client.api.data_classes.time_series import Dimension, TimeSeries, Unit, Units +from exabel_data_sdk.client.api.entity_api import EntityApi +from exabel_data_sdk.client.api.search_service import SearchService +from exabel_data_sdk.client.exabel_client import ExabelClient from exabel_data_sdk.services.file_time_series_parser import ( + MetaDataSignalNamesInRows, ParsedTimeSeriesFile, SignalNamesInColumns, SignalNamesInRows, TimeSeriesFileParser, _remove_dot_int, ) -from exabel_data_sdk.util.resource_name_normalization import EntityResourceNames +from exabel_data_sdk.stubs.exabel.api.data.v1.all_pb2 import ( + Entity, + SearchEntitiesResponse, + SearchTerm, +) +from exabel_data_sdk.util.resource_name_normalization import ( + EntityResourceNames, + EntitySearchResultWarning, +) # pylint: disable=protected-access +BLOOMBERG_TICKER_MAPPING = {"AAPL US": "F_000C7F-E", "MSFT US": "F_000Q07-E"} + class TestTimeSeriesFileParser(unittest.TestCase): def test_from_file__excel_with_batch_size_should_fail(self): @@ -218,3 +235,116 @@ def test_get_series(self): ] for e in expected: pd.testing.assert_series_equal(e, name_to_series[e.name]) + + def _entities_by_terms( + self, entity_type: str, terms: Sequence[SearchTerm] + ) -> Sequence[SearchEntitiesResponse.SearchResult]: + results = [] + for term in terms: + entities = [] + if BLOOMBERG_TICKER_MAPPING.get(term.query): + entities.append( + Entity( + name=f"{entity_type}/entities/{BLOOMBERG_TICKER_MAPPING[term.query]}", + display_name=term.query, + ) + ) + results.append( + SearchEntitiesResponse.SearchResult( + terms=[term], + entities=entities, + ) + ) + + return results + + def test_from_data_frame_long_and_medium_format(self): + parsers = [SignalNamesInColumns, SignalNamesInRows] + data = pd.DataFrame( + { + "entity": ["AAPL US", "AAPL US", "MSFT US", "unmapped"], + "date": ["2021-01-01", "2021-01-02", "2021-01-01", "2021-01-03"], + "signal": [1, 2, 4, 5], + } + ) + + client = mock.create_autospec(ExabelClient) + client.entity_api = mock.create_autospec(EntityApi) + client.entity_api.search = mock.create_autospec(SearchService) + client.entity_api.search.entities_by_terms.side_effect = self._entities_by_terms + + for parser in parsers: + if parser == SignalNamesInRows: + data = data.rename(columns={"signal": "value"}) + data.loc[:, "signal"] = "signal" + parsed_file = parser.from_data_frame( + data.copy(), + client.entity_api, + namespace="ns", + entity_type="company", + identifier_type="bloomberg_ticker", + ) + + expected_data = data.copy() + expected_data["entity"] = data["entity"].apply( + lambda x: ( + f"entityTypes/company/entities/{BLOOMBERG_TICKER_MAPPING[x]}" + if BLOOMBERG_TICKER_MAPPING.get(x) + else None + ) + ) + expected_data = expected_data.set_index("date", drop=True) + expected_data.index = pd.DatetimeIndex(expected_data.index, tz=tz.tzutc()) + expected_data.index.name = None + expected_data = expected_data.dropna(subset=["entity"], how="any") + pdt.assert_frame_equal(parsed_file.data, expected_data) + + lookup_result = parsed_file.get_entity_lookup_result() + self.assertDictEqual( + lookup_result.mapping, + { + k: f"entityTypes/company/entities/{v}" + for k, v in BLOOMBERG_TICKER_MAPPING.items() + }, + ) + self.assertListEqual( + lookup_result.warnings, + [ + EntitySearchResultWarning( + field="bloomberg_ticker", query="unmapped", matched_entity_names=[] + ) + ], + ) + + +class TestMetaDataSignalNamesInRows(unittest.TestCase): + def test_get_deduplicated_series(self): + series = [ + TimeSeries( + pd.Series([], name="time_series_1", dtype=object), + units=Units(units=[Unit(Dimension.DIMENSION_CURRENCY, unit="USD")]), + ), + TimeSeries( + pd.Series([], name="time_series_1", dtype=object), + units=Units(units=[Unit(Dimension.DIMENSION_CURRENCY, unit="USD")]), + ), + TimeSeries( + pd.Series([], name="time_series_2", dtype=object), + units=Units(units=[Unit(Dimension.DIMENSION_CURRENCY, unit="USD")]), + ), + TimeSeries( + pd.Series([], name="time_series_2", dtype=object), + units=Units(units=[Unit(Dimension.DIMENSION_CURRENCY, unit="GBP")]), + ), + ] + logger = "exabel_data_sdk.services.file_time_series_parser" + with self.assertLogs(logger, level="ERROR") as log: + series_deduplicated, series_with_duplicate_names = ( + MetaDataSignalNamesInRows._get_deduplicated_series(series) + ) + self.assertIn( + "Time series time_series_2 detected with the following different metadata", + log.output[0], + ) + self.assertListEqual([series[0]], series_deduplicated) + self.assertListEqual(series[2:], series_with_duplicate_names) diff --git a/exabel_data_sdk/tests/util/test_deprecate_arguments.py b/exabel_data_sdk/tests/util/test_deprecate_arguments.py index 2bd33c9..544d056 100644 --- a/exabel_data_sdk/tests/util/test_deprecate_arguments.py +++ b/exabel_data_sdk/tests/util/test_deprecate_arguments.py @@ -58,8 +58,7 @@ def test_deprecate_argument__no_deprecations_provided_should_fail(self): with self.assertRaises(ValueError): @deprecate_arguments() - def _no_deprecation() -> None: - ... + def _no_deprecation() -> None: ... def test_deprecate_argument__removed_argument(self): @deprecate_arguments(deprecated_arg=None) diff --git a/exabel_data_sdk/tests/util/test_logging_thread_pool_executor.py b/exabel_data_sdk/tests/util/test_logging_thread_pool_executor.py new file mode 100644 index 0000000..6484fb0 --- /dev/null +++ b/exabel_data_sdk/tests/util/test_logging_thread_pool_executor.py @@ -0,0 +1,30 @@ +import threading +import unittest + +from exabel_data_sdk.util.logging_thread_pool_executor import LoggingThreadPoolExecutor + + +class TestLoggingThreadPoolExecutor(unittest.TestCase): + def test_thread_count_with_multiple_tasks(self): + """Test that the running thread count is correctly managed.""" + executor = LoggingThreadPoolExecutor(max_workers=2) + + start_semaphore = threading.Semaphore(0) + start_barrier = threading.Barrier(2 + 1) # For 2 worker threads + 1 main test thread + + def task(): + start_semaphore.release() # Signal that the task has started + start_barrier.wait() # Wait for all other threads to reach this point + return "result" + + futures = [executor.submit(task) for _ in range(2)] + + for _ in range(2): + start_semaphore.acquire() # pylint: disable=consider-using-with + self.assertEqual(executor.running_threads, 2) + + start_barrier.wait() + + result = [future.result() for future in futures] + self.assertEqual(executor.running_threads, 0) + self.assertListEqual(result, ["result"] * 2) diff --git a/exabel_data_sdk/util/deprecate_arguments.py b/exabel_data_sdk/util/deprecate_arguments.py index b20609a..dd64dc8 100644 --- a/exabel_data_sdk/util/deprecate_arguments.py +++ b/exabel_data_sdk/util/deprecate_arguments.py @@ -10,24 +10,21 @@ @overload def deprecate_arguments( **deprecation_replacements: Optional[str], -) -> Callable[[FunctionT], FunctionT]: - ... +) -> Callable[[FunctionT], FunctionT]: ... @overload def deprecate_arguments( __func: None, # pylint: disable=invalid-name **deprecation_replacements: Optional[str], -) -> Callable[[FunctionT], FunctionT]: - ... +) -> Callable[[FunctionT], FunctionT]: ... @overload def deprecate_arguments( __func: FunctionT, # pylint: disable=invalid-name **deprecation_replacements: Optional[str], -) -> FunctionT: - ... +) -> FunctionT: ... # Pylint flags '__func' as an invalid argument name, but we want the '__' prefix to make Mypy @@ -93,24 +90,21 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: @overload def deprecate_argument_value( **deprecated_values: object, -) -> Callable[[FunctionT], FunctionT]: - ... +) -> Callable[[FunctionT], FunctionT]: ... @overload def deprecate_argument_value( __func: None, # pylint: disable=invalid-name **deprecated_values: object, -) -> Callable[[FunctionT], FunctionT]: - ... +) -> Callable[[FunctionT], FunctionT]: ... @overload def deprecate_argument_value( __func: FunctionT, # pylint: disable=invalid-name **deprecated_values: object, -) -> FunctionT: - ... +) -> FunctionT: ... def deprecate_argument_value( diff --git a/exabel_data_sdk/util/logging_thread_pool_executor.py b/exabel_data_sdk/util/logging_thread_pool_executor.py new file mode 100644 index 0000000..a127284 --- /dev/null +++ b/exabel_data_sdk/util/logging_thread_pool_executor.py @@ -0,0 +1,40 @@ +import logging +import threading +from concurrent.futures import Future, ThreadPoolExecutor +from typing import Any, Callable + +logger = logging.getLogger(__name__) + + +class LoggingThreadPoolExecutor(ThreadPoolExecutor): + """ + A thread pool executor logs the number of active threads + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + self.running_threads = 0 + self.lock = threading.Lock() + super().__init__(*args, **kwargs) + + def active_threads_counter(self, func: Callable) -> Callable: + """Decorator to count the number of active threads.""" + + def wrapped(*args, **kwargs): # type: ignore + with self.lock: + self.running_threads += 1 + logger.debug("Thread started. Active threads: %d", self.running_threads) + try: + result = func(*args, **kwargs) + finally: + with self.lock: + self.running_threads -= 1 + logger.debug("Thread finished. Active threads: %d", self.running_threads) + + return result + + return wrapped + + def submit( # type: ignore[override] # pylint: disable=arguments-differ + self, function: Callable[..., Any], *args: Any, **kwargs: Any + ) -> Future: + return super().submit(self.active_threads_counter(function), *args, **kwargs)