try this fix

danielgafni · Nov 1, 2023 · 04e4bfd · 04e4bfd
1 parent 272d748
commit 04e4bfd
Show file tree

Hide file tree

Showing 4 changed files with 31 additions and 51 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -21,19 +21,14 @@ repos:
 #        entry: pyupgrade --py38-plus
   - repo: local
     hooks:
-      - id: isort
-        name: isort
-        entry: isort --settings-file pyproject.toml .
-        language: system
-        pass_filenames: false
-      - id: ruff
-        name: ruff
+      - id: lint
+        name: lint
         entry: ruff --fix .
         language: system
         pass_filenames: false
-      - id: black
-        name: black
-        entry: black .
+      - id: format
+        name: format
+        entry: ruff format .
         language: system
         pass_filenames: false
       - id: black-docs

diff --git a/dagster_polars/io_managers/delta.py b/dagster_polars/io_managers/delta.py
@@ -59,7 +59,7 @@ def dump_df_to_path(self, context: OutputContext, df: pl.DataFrame, path: UPath)
             storage_options=storage_options,
             delta_write_options=delta_write_options,
         )
-        table = DeltaTable(str(path), storage_options=storage_options or None)
+        table = DeltaTable(str(path), storage_options={k.upper(): v for k, v in self.get_storage_options(path).items()})
         context.add_output_metadata({"version": table.version()})
 
     def scan_df_from_path(self, path: UPath, context: InputContext) -> pl.LazyFrame:
@@ -83,7 +83,7 @@ def scan_df_from_path(self, path: UPath, context: InputContext) -> pl.LazyFrame:
 
         version = DeltaTable(
             str(path),
-            storage_options=self.get_storage_options(path) or None,
+            storage_options={k.upper(): v for k, v in self.get_storage_options(path).items()},
             version=version,
         ).version()
 
@@ -144,7 +144,9 @@ def get_metadata(self, context: OutputContext, obj: pl.DataFrame) -> Dict[str, M
                 path = self._get_path(context)
                 # we need to get row_count from the full table
                 metadata["row_count"] = MetadataValue.int(
-                    DeltaTable(str(path), storage_options=self.get_storage_options(path) or None)
+                    DeltaTable(
+                        str(path), storage_options={k.upper(): v for k, v in self.get_storage_options(path).items()}
+                    )
                     .to_pyarrow_dataset()
                     .count_rows()
                 )

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -46,8 +46,7 @@ deltalake = ["deltalake"]
 hypothesis = "^6.77.0"
 pytest = "^7.3.1"
 deepdiff = "^6.3.0"
-isort = "^5.12.0"
-ruff = "^0.0.272"
+ruff = "^0.1.3"
 pyright = "^1.1.313"
 tox = "^4.6.0"
 tox-gh = "^1.0.0"
@@ -105,6 +104,7 @@ exclude = '''
 '''
 
 [tool.ruff]
+extend-select = ["I"]
 line-length = 120
 src = [
     "dagster_polars",