Merge pull request #23 from regro/dep-libcfgraph

conda_forge_metadata/artifact_info/info_json.py

@@ -2,21 +2,22 @@
 
 import json
 import tarfile
+import warnings
 from typing import Any, Generator, Tuple
 
 from ruamel import yaml
 
 from conda_forge_metadata.libcfgraph import get_libcfgraph_artifact_data
 from conda_forge_metadata.types import ArtifactData
 
-VALID_BACKENDS = ("libcfgraph", "oci", "streamed")
+VALID_BACKENDS = ("oci", "streamed")
 
 
 def get_artifact_info_as_json(
     channel: str,
     subdir: str,
     artifact: str,
-    backend: str = "libcfgraph",
+    backend: str = "oci",
     skip_files_suffixes: Tuple[str, ...] = (".pyc", ".txt"),
 ) -> ArtifactData | None:
     """Get a blob of artifact data from the conda info directory.
@@ -30,6 +31,12 @@ def get_artifact_info_as_json(
     artifact : str
         The full artifact name with extension (e.g.,
         "21cmfast-3.0.2-py36h13dd421_0.tar.bz2").
+    backend : str, optional
+        The backend information source to use for the metadata. Valid
+        backends are "oci" and "streamed". The default is "oci".
+    skip_files_suffixes : Tuple[str, ...], optional
+        A tuple of suffixes to skip when reporting the files in the
+        artifact. The default is (".pyc", ".txt").
 
     Returns
     -------
@@ -53,6 +60,14 @@ def get_artifact_info_as_json(
                 elements ending in .pyc or .txt filtered out.
     """
     if backend == "libcfgraph":
+        warnings.simplefilter("always", DeprecationWarning)
+        warnings.warn(
+            "The 'libcfgraph' backend for get_artifact_info_as_json is deprecated and "
+            "will be removed in a future release. Use 'oci' or 'streamed' instead.",
+            category=DeprecationWarning,
+            stacklevel=2,
+        )
+        warnings.simplefilter("default", DeprecationWarning)
         return get_libcfgraph_artifact_data(channel, subdir, artifact)
     elif backend == "oci":
         from conda_forge_metadata.oci import get_oci_artifact_data

conda_forge_metadata/autotick_bot/__init__.py

@@ -1,2 +1,2 @@
-from .import_to_pkg import map_import_to_package  # noqa
+from .import_to_pkg import get_pkgs_for_import, map_import_to_package  # noqa
 from .pypi_to_conda import get_pypi_name_mapping, map_pypi_to_conda  # noqa

conda_forge_metadata/autotick_bot/import_to_pkg.py

@@ -1,10 +1,94 @@
 from __future__ import annotations
 
+import hashlib
+import os
 from functools import lru_cache
 
 import requests
 
-from conda_forge_metadata.libcfgraph import get_libcfgraph_pkgs_for_import
+AUTOTICK_BOT_GITHUB_BASE_URL = "https://github.com/regro/cf-graph-countyfair/raw/master"
+
+
+@lru_cache(maxsize=1)
+def _import_to_pkg_maps_num_letters() -> int:
+    req = requests.get(
+        f"{AUTOTICK_BOT_GITHUB_BASE_URL}"
+        "/import_to_pkg_maps/import_to_pkg_maps_meta.json"
+    )
+    req.raise_for_status()
+    return int(req.json()["num_letters"])
+
+
+@lru_cache(maxsize=1)
+def _import_to_pkg_maps_num_dirs() -> int:
+    req = requests.get(
+        f"{AUTOTICK_BOT_GITHUB_BASE_URL}"
+        "/import_to_pkg_maps/import_to_pkg_maps_meta.json"
+    )
+    req.raise_for_status()
+    return int(req.json()["num_dirs"])
+
+
+def _get_bot_sharded_path(file_path, n_dirs=5):
+    """computed a sharded location for the LazyJson file."""
+    top_dir, file_name = os.path.split(file_path)
+
+    if len(top_dir) == 0 or top_dir == "lazy_json":
+        return file_name
+    else:
+        hx = hashlib.sha1(file_name.encode("utf-8")).hexdigest()[0:n_dirs]
+        pth_parts = [top_dir] + [hx[i] for i in range(n_dirs)] + [file_name]
+        return os.path.join(*pth_parts)
+
+
+@lru_cache(maxsize=128)
+def _import_to_pkg_maps_cache(import_first_letters: str) -> dict[str, set[str]]:
+    pth = _get_bot_sharded_path(
+        f"import_to_pkg_maps/{import_first_letters.lower()}.json",
+        n_dirs=_import_to_pkg_maps_num_dirs(),
+    )
+    req = requests.get(f"{AUTOTICK_BOT_GITHUB_BASE_URL}/{pth}")
+    req.raise_for_status()
+    return {k: set(v["elements"]) for k, v in req.json().items()}
+
+
+def _get_pkgs_for_import(import_name: str) -> set[str] | None:
+    num_letters = _import_to_pkg_maps_num_letters()
+    fllt = import_name[: min(len(import_name), num_letters)]
+    import_to_pkg_map = _import_to_pkg_maps_cache(fllt)
+    return import_to_pkg_map.get(import_name, None)
+
+
+def get_pkgs_for_import(import_name: str) -> tuple[set[str] | None, str]:
+    """Get a list of possible packages that supply a given import.
+
+    **This data is approximate and may be wrong.**
+
+    For a better guess, use the function
+
+        `conda_forge_metadata.autotick_bot.map_import_to_package`
+
+    which attempts to return the most likely supplying package.
+
+    Parameters
+    ----------
+    import_name : str
+        The name of the import.
+
+    Returns
+    -------
+    packages : set
+        A set of packages that possibly have the import.
+        Will return `None` if the import was not found.
+    found_import_name : str
+        The import name found in the libcfgraph metadata. Only
+        valid if `packages` is not None. This name will be the
+        top-level import with all subpackages removed (e.g., foo.bar.baz
+        will be returned as foo).
+    """
+    import_name = import_name.split(".")[0]
+    supplying_pkgs = _get_pkgs_for_import(import_name)
+    return supplying_pkgs, import_name
 
 
 @lru_cache(maxsize=1)
@@ -30,7 +114,7 @@ def map_import_to_package(import_name: str) -> str:
     pkg_name : str
         The name of the package.
     """
-    supplying_pkgs, found_import_name = get_libcfgraph_pkgs_for_import(import_name)
+    supplying_pkgs, found_import_name = get_pkgs_for_import(import_name)
     if supplying_pkgs is None:
         return found_import_name
 

conda_forge_metadata/libcfgraph.py

@@ -3,6 +3,7 @@
 from functools import lru_cache
 
 import requests
+from deprecated import deprecated
 
 from conda_forge_metadata.types import ArtifactData
 
@@ -27,6 +28,12 @@ def _download_libcfgraph_index():
         _LIBCFGRAPH_INDEX += r.json()
 
 
+@deprecated(
+    reason=(
+        "conda-forge no longer maintains the libcfgraph metadata store. "
+        "This API will be removed in a future release."
+    )
+)
 def get_libcfgraph_index() -> list[str]:
     """Get a list of all artifacts indexed by libcfgraph.
 
@@ -44,6 +51,13 @@ def get_libcfgraph_index() -> list[str]:
     return _LIBCFGRAPH_INDEX
 
 
+@deprecated(
+    reason=(
+        "conda-forge no longer maintains the libcfgraph metadata store. "
+        "This API will be removed in a future release. "
+        "Use conda_forge_metdata.artifact_info.get_artifact_info_as_json instead."
+    )
+)
 @lru_cache(maxsize=1024)
 def get_libcfgraph_artifact_data(
     channel: str, subdir: str, artifact: str
@@ -132,6 +146,13 @@ def _get_libcfgraph_pkgs_for_import(import_name: str) -> set[str] | None:
     return import_to_pkg_map.get(import_name, None)
 
 
+@deprecated(
+    reason=(
+        "conda-forge no longer maintains the libcfgraph metadata store. "
+        "This API will be removed in a future release. "
+        "Use conda_forge_metdata.autotick_bot.get_pks_for_import instead."
+    )
+)
 def get_libcfgraph_pkgs_for_import(import_name: str) -> tuple[set[str] | None, str]:
     """Get a list of possible packages that supply a given import.
 

pyproject.toml

@@ -14,6 +14,7 @@ authors = [
 description = "programatic access to conda-forge's metadata"
 dynamic = ["version"]
 dependencies = [
+    "deprecated",
     "requests",
     "ruamel.yaml",
     "typing-extensions"

requirements.txt

@@ -1,4 +1,5 @@
 conda-oci-mirror
+deprecated
 requests
 ruamel.yaml
 typing-extensions

tests/test_map_import_to_package.py

@@ -1,4 +1,4 @@
-from conda_forge_metadata.autotick_bot import map_import_to_package
+from conda_forge_metadata.autotick_bot import get_pkgs_for_import, map_import_to_package
 
 
 def test_map_import_to_package():
@@ -9,3 +9,27 @@ def test_map_import_to_package():
     assert map_import_to_package("eastlake") == "des-eastlake"
 
     assert map_import_to_package("scipy") == "scipy"
+
+
+def test_get_pkgs_for_import():
+    pkgs, nm = get_pkgs_for_import("numpy")
+    assert pkgs is not None
+    assert nm == "numpy"
+    assert "numpy" in pkgs
+
+    pkgs, nm = get_pkgs_for_import("numpy.linalg")
+    assert pkgs is not None
+    assert nm == "numpy"
+    assert "numpy" in pkgs
+
+    # something bespoke
+    pkgs, nm = get_pkgs_for_import("eastlake")
+    assert pkgs is not None
+    assert nm == "eastlake"
+    assert "des-eastlake" in pkgs
+    assert len(pkgs) == 1
+
+    pkgs, nm = get_pkgs_for_import("scipy")
+    assert pkgs is not None
+    assert nm == "scipy"
+    assert "scipy" in pkgs