Skip to content
This repository has been archived by the owner on Mar 11, 2024. It is now read-only.

Commit

Permalink
fix overloads
Browse files Browse the repository at this point in the history
  • Loading branch information
ion-elgreco committed Jan 28, 2024
1 parent 66f47a1 commit 2e33cb8
Show file tree
Hide file tree
Showing 2 changed files with 29 additions and 5 deletions.
14 changes: 13 additions & 1 deletion dagster_polars/io_managers/delta.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import json
from enum import Enum
from pprint import pformat
from typing import TYPE_CHECKING, Any, Dict, Optional, Union
from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Union, overload

import dagster._check as check
import polars as pl
Expand Down Expand Up @@ -241,6 +241,18 @@ def write_df_to_path(
metadata_path.parent.mkdir(parents=True, exist_ok=True)
metadata_path.write_text(json.dumps(metadata))

@overload
def scan_df_from_path(
self, path: "UPath", context: InputContext, with_metadata: Literal[None, False]
) -> pl.LazyFrame:
...

@overload
def scan_df_from_path(
self, path: "UPath", context: InputContext, with_metadata: Literal[True]
) -> LazyFrameWithMetadata:
...

def scan_df_from_path(
self,
path: "UPath",
Expand Down
20 changes: 16 additions & 4 deletions dagster_polars/io_managers/parquet.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import json
from typing import TYPE_CHECKING, Any, Optional, Union
from typing import TYPE_CHECKING, Any, Literal, Optional, Union, overload

import polars as pl
import pyarrow.dataset as ds
Expand Down Expand Up @@ -145,7 +145,7 @@ def downsteam(upstream: DataFrameWithMetadata):
assert metadata["my_custom_metadata"] == "my_custom_value"
"""

extension: str = ".parquet"
extension: str = ".parquet" # type: ignore

def sink_df_to_path(
self,
Expand Down Expand Up @@ -256,12 +256,24 @@ def write_df_to_path(
row_group_size=row_group_size,
)

def scan_df_from_path( # type: ignore
@overload
def scan_df_from_path(
self, path: "UPath", context: InputContext, with_metadata: Literal[None, False]
) -> pl.LazyFrame:
...

@overload
def scan_df_from_path(
self, path: "UPath", context: InputContext, with_metadata: Literal[True]
) -> LazyFrameWithMetadata:
...

def scan_df_from_path(
self,
path: "UPath",
context: InputContext,
partition_key: Optional[str] = None,
with_metadata: Optional[bool] = False,
partition_key: Optional[str] = None,
) -> Union[pl.LazyFrame, LazyFrameWithMetadata]:
ldf = scan_parquet(path, context)

Expand Down

0 comments on commit 2e33cb8

Please sign in to comment.