Adds a pipenv_requirements macro (#10654)

Part of #10655.

This macro will read a `Pipfile.lock` and convert everything into a `python_requirement_library`. It won't actually treat this as a true lockfile, i.e. it won't feed the results into the option `--python-setup-requirements-constraints`. That will need to come in a followup. Rather, this acts like the macro `python_requirements()`, just with a Pipfile.lock.

src/python/pants/backend/python/pipenv_requirements.py

@@ -0,0 +1,88 @@
+# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from json import load
+from pathlib import Path
+from typing import Iterable, Mapping, Optional
+
+from pkg_resources import Requirement
+
+from pants.base.build_environment import get_buildroot
+
+
+class PipenvRequirements:
+    """Translates a Pipenv.lock file into an equivalent set `python_requirement_library` targets.
+
+    You may also use the parameter `module_mapping` to teach Pants what modules each of your
+    requirements provide. For any requirement unspecified, Pants will default to the name of the
+    requirement. This setting is important for Pants to know how to convert your import
+    statements back into your dependencies. For example:
+
+        pipenv_requirements(
+          module_mapping={
+            "ansicolors": ["colors"],
+            "setuptools": ["pkg_resources"],
+          }
+        )
+    """
+
+    def __init__(self, parse_context):
+        self._parse_context = parse_context
+
+    def __call__(
+        self,
+        requirements_relpath: str = "Pipfile.lock",
+        module_mapping: Optional[Mapping[str, Iterable[str]]] = None,
+        pipfile_target: Optional[str] = None,
+    ) -> None:
+        """
+        :param requirements_relpath: The relpath from this BUILD file to the requirements file.
+            Defaults to a `Pipfile.lock` file sibling to the BUILD file.
+        :param module_mapping: a mapping of requirement names to a list of the modules they provide.
+            For example, `{"ansicolors": ["colors"]}`. Any unspecified requirements will use the
+            requirement name as the default module, e.g. "Django" will default to
+            `modules=["django"]`.
+        :param pipfile_target: a `_python_requirements_file` target to provide for cache invalidation
+        if the requirements_relpath value is not in the current rel_path
+        """
+
+        lock_info = {}
+
+        requirements_path = Path(
+            get_buildroot(), self._parse_context.rel_path, requirements_relpath
+        )
+        with open(requirements_path, "r") as fp:
+            lock_info = load(fp)
+
+        if pipfile_target:
+            requirements_dep = pipfile_target
+        else:
+            requirements_file_target_name = requirements_relpath
+            self._parse_context.create_object(
+                "_python_requirements_file",
+                name=requirements_file_target_name,
+                sources=[requirements_relpath],
+            )
+            requirements_dep = f":{requirements_file_target_name}"
+
+        requirements = {**lock_info.get("default", {}), **lock_info.get("develop", {})}
+        for req, info in requirements.items():
+            req_str = f"{req}{info.get('version','')}"
+            if info.get("markers"):
+                req_str += f";{info['markers']}"
+
+            parsed_req = Requirement.parse(req_str)
+
+            req_module_mapping = (
+                {parsed_req.project_name: module_mapping[parsed_req.project_name]}
+                if module_mapping and parsed_req.project_name in module_mapping
+                else None
+            )
+
+            self._parse_context.create_object(
+                "python_requirement_library",
+                name=parsed_req.project_name,
+                requirements=[parsed_req],
+                dependencies=[requirements_dep],
+                module_mapping=req_module_mapping,
+            )

src/python/pants/backend/python/pipenv_requirements_test.py

@@ -0,0 +1,126 @@
+# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from json import dumps
+from textwrap import dedent
+from typing import Iterable
+
+from pkg_resources import Requirement
+
+from pants.backend.python.pipenv_requirements import PipenvRequirements
+from pants.backend.python.target_types import PythonRequirementLibrary, PythonRequirementsFile
+from pants.base.specs import AddressSpecs, DescendantAddresses, FilesystemSpecs, Specs
+from pants.build_graph.build_file_aliases import BuildFileAliases
+from pants.engine.addresses import Address
+from pants.engine.target import Targets
+from pants.testutil.engine.util import Params
+from pants.testutil.option.util import create_options_bootstrapper
+from pants.testutil.test_base import TestBase
+
+
+class PipenvRequirementsTest(TestBase):
+    @classmethod
+    def alias_groups(cls):
+        return BuildFileAliases(
+            context_aware_object_factories={"pipenv_requirements": PipenvRequirements},
+        )
+
+    @classmethod
+    def target_types(cls):
+        return [PythonRequirementLibrary, PythonRequirementsFile]
+
+    def assert_pipenv_requirements(
+        self,
+        build_file_entry: str,
+        pipfile_lock: dict,
+        *,
+        expected_file_dep: PythonRequirementsFile,
+        expected_targets: Iterable[PythonRequirementLibrary],
+        pipfile_lock_relpath: str = "Pipfile.lock",
+    ) -> None:
+        self.add_to_build_file("", f"{build_file_entry}\n")
+        self.create_file(pipfile_lock_relpath, dumps(pipfile_lock))
+        targets = self.request_single_product(
+            Targets,
+            Params(
+                Specs(AddressSpecs([DescendantAddresses("")]), FilesystemSpecs([])),
+                create_options_bootstrapper(),
+            ),
+        )
+
+        assert {expected_file_dep, *expected_targets} == set(targets)
+
+    def test_pipfile_lock(self) -> None:
+        """This tests that we correctly create a new python_requirement_library for each entry in a
+        Pipfile.lock file.
+
+        Edge cases:
+        * Develop and Default requirements are used
+        * If a module_mapping is given, and the project is in the map, we copy over a subset of the mapping to the created target.
+        """
+
+        self.assert_pipenv_requirements(
+            "pipenv_requirements(module_mapping={'ansicolors': ['colors']})",
+            {
+                "default": {"ansicolors": {"version": ">=1.18.0"},},
+                "develop": {
+                    "cachetools": {"markers": "python_version ~= '3.5'", "version": "==4.1.1"},
+                },
+            },
+            expected_file_dep=PythonRequirementsFile(
+                {"sources": ["Pipfile.lock"]}, address=Address("", target_name="Pipfile.lock")
+            ),
+            expected_targets=[
+                PythonRequirementLibrary(
+                    {
+                        "requirements": [Requirement.parse("ansicolors>=1.18.0")],
+                        "dependencies": [":Pipfile.lock"],
+                        "module_mapping": {"ansicolors": ["colors"]},
+                    },
+                    address=Address("", target_name="ansicolors"),
+                ),
+                PythonRequirementLibrary(
+                    {
+                        "requirements": [
+                            Requirement.parse("cachetools==4.1.1;python_version ~= '3.5'")
+                        ],
+                        "dependencies": [":Pipfile.lock"],
+                    },
+                    address=Address("", target_name="cachetools"),
+                ),
+            ],
+        )
+
+    def test_supply_python_requirements_file(self) -> None:
+        """This tests that we can supply our own `_python_requirements_file`."""
+
+        self.assert_pipenv_requirements(
+            dedent(
+                """
+            pipenv_requirements(
+                requirements_relpath='custom/pipfile/Pipfile.lock',
+                pipfile_target='//:custom_pipfile_target'
+            )
+
+            _python_requirements_file(
+                name='custom_pipfile_target',
+                sources=['custom/pipfile/Pipfile.lock']
+            )
+            """
+            ),
+            {"default": {"ansicolors": {"version": ">=1.18.0"},},},
+            expected_file_dep=PythonRequirementsFile(
+                {"sources": ["custom/pipfile/Pipfile.lock"]},
+                address=Address("", target_name="custom_pipfile_target"),
+            ),
+            expected_targets=[
+                PythonRequirementLibrary(
+                    {
+                        "requirements": [Requirement.parse("ansicolors>=1.18.0")],
+                        "dependencies": ["//:custom_pipfile_target"],
+                    },
+                    address=Address("", target_name="ansicolors"),
+                ),
+            ],
+            pipfile_lock_relpath="custom/pipfile/Pipfile.lock",
+        )

src/python/pants/backend/python/register.py

@@ -8,6 +8,7 @@
 
 from pants.backend.python.dependency_inference import rules as dependency_inference_rules
 from pants.backend.python.pants_requirement import PantsRequirement
+from pants.backend.python.pipenv_requirements import PipenvRequirements
 from pants.backend.python.python_artifact import PythonArtifact
 from pants.backend.python.python_requirements import PythonRequirements
 from pants.backend.python.rules import (
@@ -46,6 +47,7 @@ def build_file_aliases():
         },
         context_aware_object_factories={
             "python_requirements": PythonRequirements,
+            "pipenv_requirements": PipenvRequirements,
             PantsRequirement.alias: PantsRequirement,
         },
     )