Update sdk packages (rtdip#508)

* Update delta sharing package Signed-off-by: GBBBAS <[email protected]> * Updates to packages Signed-off-by: GBBBAS <[email protected]> * Update to test Signed-off-by: GBBBAS <[email protected]> --------- Signed-off-by: GBBBAS <[email protected]>
GBBBAS · Sep 26, 2023 · 82f4c35 · 82f4c35
1 parent 27f8118
commit 82f4c35
Show file tree

Hide file tree

Showing 4 changed files with 6 additions and 7 deletions.
diff --git a/environment.yml b/environment.yml
@@ -60,7 +60,7 @@ dependencies:
     - strawberry-graphql[fastapi,pydantic]==0.194.4
     - web3==6.5.0
     - twine==4.0.2
-    - delta-sharing-python==0.7.4
+    - delta-sharing-python==1.0.0
     - polars==0.18.8
     - moto[s3]==4.1.14
     - xarray>=2023.1.0,<2023.8.0

diff --git a/setup.py b/setup.py
@@ -58,7 +58,7 @@
     "azure-keyvault-secrets==4.7.0",
     "web3==6.5.0",
     "polars[deltalake]==0.18.8",
-    "delta-sharing==0.7.4",
+    "delta-sharing==1.0.0",
     "xarray>=2023.1.0,<2023.8.0",
     "ecmwf-api-client==1.6.3",
     "netCDF4==1.6.4",

diff --git a/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/constants.py b/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/constants.py
@@ -35,7 +35,7 @@ def get_default_package(package_name):
             version=_get_package_version("delta-spark"),
         ),
         "spark_delta_sharing": MavenLibrary(
-            group_id="io.delta", artifact_id="delta-sharing-spark_2.12", version="0.6.3"
+            group_id="io.delta", artifact_id="delta-sharing-spark_2.12", version="1.0.0"
         ),
         "spark_azure_eventhub": MavenLibrary(
             group_id="com.microsoft.azure",
@@ -52,14 +52,13 @@ def get_default_package(package_name):
             artifact_id="spark-connect_2.12",
             version=_get_package_version("pyspark"),
         ),
-        "rtdip_sdk": PyPiLibrary(name="rtdip_sdk", version="0.5.1"),
         "azure_adls_gen_2": PyPiLibrary(
-            name="azure-storage-file-datalake", version="12.10.1"
+            name="azure-storage-file-datalake", version="12.12.0"
         ),
         "azure_key_vault_secret": PyPiLibrary(
             name="azure-keyvault-secrets", version="4.7.0"
         ),
-        "aws_boto3": PyPiLibrary(name="boto3", version="1.26.118"),
+        "aws_boto3": PyPiLibrary(name="boto3", version="1.28.2"),
         "hashicorp_vault": PyPiLibrary(name="hvac", version="1.1.0"),
         "api_requests": PyPiLibrary(name="requests", version="2.30.0"),
         "pyarrow": PyPiLibrary(name="pyarrow", version="12.0.0"),

diff --git a/tests/sdk/python/rtdip_sdk/pipelines/sources/spark/test_delta_sharing.py b/tests/sdk/python/rtdip_sdk/pipelines/sources/spark/test_delta_sharing.py
@@ -41,7 +41,7 @@ def test_spark_delta_sharing_read_setup(spark_session: SparkSession):
             MavenLibrary(
                 group_id="io.delta",
                 artifact_id="delta-sharing-spark_2.12",
-                version="0.6.3",
+                version="1.0.0",
             )
         ],
         pypi_libraries=[],