diff --git a/environment.yml b/environment.yml index 32989399e..21e296f53 100644 --- a/environment.yml +++ b/environment.yml @@ -60,7 +60,7 @@ dependencies: - strawberry-graphql[fastapi,pydantic]==0.194.4 - web3==6.5.0 - twine==4.0.2 - - delta-sharing-python==0.7.4 + - delta-sharing-python==1.0.0 - polars==0.18.8 - moto[s3]==4.1.14 - xarray>=2023.1.0,<2023.8.0 diff --git a/setup.py b/setup.py index e12e27b37..fb321419d 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ "azure-keyvault-secrets==4.7.0", "web3==6.5.0", "polars[deltalake]==0.18.8", - "delta-sharing==0.7.4", + "delta-sharing==1.0.0", "xarray>=2023.1.0,<2023.8.0", "ecmwf-api-client==1.6.3", "netCDF4==1.6.4", diff --git a/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/constants.py b/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/constants.py index fed2806b1..877638d07 100644 --- a/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/constants.py +++ b/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/constants.py @@ -35,7 +35,7 @@ def get_default_package(package_name): version=_get_package_version("delta-spark"), ), "spark_delta_sharing": MavenLibrary( - group_id="io.delta", artifact_id="delta-sharing-spark_2.12", version="0.6.3" + group_id="io.delta", artifact_id="delta-sharing-spark_2.12", version="1.0.0" ), "spark_azure_eventhub": MavenLibrary( group_id="com.microsoft.azure", @@ -52,14 +52,13 @@ def get_default_package(package_name): artifact_id="spark-connect_2.12", version=_get_package_version("pyspark"), ), - "rtdip_sdk": PyPiLibrary(name="rtdip_sdk", version="0.5.1"), "azure_adls_gen_2": PyPiLibrary( - name="azure-storage-file-datalake", version="12.10.1" + name="azure-storage-file-datalake", version="12.12.0" ), "azure_key_vault_secret": PyPiLibrary( name="azure-keyvault-secrets", version="4.7.0" ), - "aws_boto3": PyPiLibrary(name="boto3", version="1.26.118"), + "aws_boto3": PyPiLibrary(name="boto3", version="1.28.2"), "hashicorp_vault": PyPiLibrary(name="hvac", version="1.1.0"), "api_requests": PyPiLibrary(name="requests", version="2.30.0"), "pyarrow": PyPiLibrary(name="pyarrow", version="12.0.0"), diff --git a/tests/sdk/python/rtdip_sdk/pipelines/sources/spark/test_delta_sharing.py b/tests/sdk/python/rtdip_sdk/pipelines/sources/spark/test_delta_sharing.py index 09d7073aa..a84159ec4 100644 --- a/tests/sdk/python/rtdip_sdk/pipelines/sources/spark/test_delta_sharing.py +++ b/tests/sdk/python/rtdip_sdk/pipelines/sources/spark/test_delta_sharing.py @@ -41,7 +41,7 @@ def test_spark_delta_sharing_read_setup(spark_session: SparkSession): MavenLibrary( group_id="io.delta", artifact_id="delta-sharing-spark_2.12", - version="0.6.3", + version="1.0.0", ) ], pypi_libraries=[],