Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Directly lookup target distributions instead of filtering manually #2120

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
+24 −27
Open

Directly lookup target distributions instead of filtering manually #2120

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d36a7c5
Directly lookup target distributions
sfc-gh-rebeling Dec 13, 2024
4019964
Fix unit tests
sfc-gh-rebeling Dec 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 11 additions & 12 deletions src/snowflake/connector/options.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import importlib
import os
import warnings
from importlib.metadata import distributions
from importlib.metadata import PackageNotFoundError, distribution
from logging import getLogger
from types import ModuleType
from typing import Union
Expand Down Expand Up @@ -85,13 +85,13 @@ def _import_or_missing_pandas_option() -> (
os.environ["ARROW_DEFAULT_MEMORY_POOL"] = "system"

# Check whether we have the currently supported pyarrow installed
installed_packages = {
package.metadata["Name"]: package for package in distributions()
}
if {"pyarrow", "snowflake-connector-python"} <= installed_packages.keys():
dependencies = installed_packages[
"snowflake-connector-python"
].metadata.get_all("Requires-Dist", [])
try:
pyarrow_dist = distribution("pyarrow")
snowflake_connector_dist = distribution("snowflake-connector-python")

dependencies = snowflake_connector_dist.metadata.get_all(
"Requires-Dist", []
)
pandas_pyarrow_extra = None
for dependency in dependencies:
dep = Requirement(dependency)
Expand All @@ -103,16 +103,15 @@ def _import_or_missing_pandas_option() -> (
pandas_pyarrow_extra = dep
break

installed_pyarrow_version = installed_packages["pyarrow"].version
installed_pyarrow_version = pyarrow_dist.version
if not pandas_pyarrow_extra.specifier.contains(installed_pyarrow_version):
warn_incompatible_dep(
"pyarrow", installed_pyarrow_version, pandas_pyarrow_extra
)

else:
except PackageNotFoundError as e:
logger.info(
"Cannot determine if compatible pyarrow is installed because of missing package(s) from "
"{}".format(list(installed_packages.keys()))
f"Cannot determine if compatible pyarrow is installed because of missing package(s): {e}"
)
return pandas, pyarrow, True
except ImportError:
Expand Down
28 changes: 13 additions & 15 deletions test/integ/pandas/test_unit_options.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
MissingPandas = None
_import_or_missing_pandas_option = None

from importlib.metadata import distributions
from importlib.metadata import PackageNotFoundError, distribution


@pytest.mark.skipif(
Expand All @@ -30,25 +30,23 @@ def test_pandas_option_reporting(caplog):

This issue was brought to attention in: https://github.com/snowflakedb/snowflake-connector-python/issues/412
"""
modified_distributions = list(
d
for d in distributions()
if d.metadata["Name"]
not in (
"pyarrow",
"snowflake-connecctor-python",
)
)

def modified_distribution(name, *args, **kwargs):
if name in ["pyarrow", "snowflake-connector-python"]:
raise PackageNotFoundError("TestErrorMessage")
return distribution(name, *args, **kwargs)

with mock.patch(
"snowflake.connector.options.distributions",
return_value=modified_distributions,
"snowflake.connector.options.distribution",
wraps=modified_distribution,
):
caplog.set_level(logging.DEBUG, "snowflake.connector")
pandas, pyarrow, installed_pandas = _import_or_missing_pandas_option()
assert installed_pandas
assert not isinstance(pandas, MissingPandas)
assert not isinstance(pyarrow, MissingPandas)
assert (
"Cannot determine if compatible pyarrow is installed because of missing package(s) "
"from "
) in caplog.text
"Cannot determine if compatible pyarrow is installed because of missing package(s)"
in caplog.text
)
assert "TestErrorMessage" in caplog.text
Loading