[components] Make the components folder parameterizable (#26619)

## Summary & Motivation

This increases the flexibility of code layout in components-enabled code location by allowing `build_components_defs` to load any arbitrary folder as the components folder. This will be useful for codebases where users have their own folder layout conventions, as well as for testing (we could have many componet folders for a single code location directory for example).

## How I Tested These Changes

Manual

python_modules/libraries/dagster-components/dagster_components/core/component.py

@@ -53,7 +53,10 @@ def get_rendering_scope(cls) -> Mapping[str, Any]:
         return {}
 
     @classmethod
-    def generate_files(cls, request: ComponentGenerateRequest, params: Any) -> None: ...
+    def generate_files(cls, request: ComponentGenerateRequest, params: Any) -> None:
+        from dagster_components.generate import generate_component_yaml
+
+        generate_component_yaml(request, {})
 
     @abstractmethod
     def build_defs(self, context: "ComponentLoadContext") -> Definitions: ...

python_modules/libraries/dagster-components/dagster_components/core/component_defs_builder.py

@@ -129,6 +129,7 @@ def build_component_defs(
     code_location_root: Path,
     resources: Optional[Mapping[str, object]] = None,
     registry: Optional["ComponentRegistry"] = None,
+    components_folder: Optional[Path] = None,
 ) -> "Definitions":
     """Build a Definitions object for all the component instances in a given code location.
 
@@ -139,7 +140,9 @@ def build_component_defs(
     from dagster._core.definitions.definitions_class import Definitions
 
     context = CodeLocationProjectContext.from_code_location_path(
-        code_location_root, registry or ComponentRegistry.from_entry_point_discovery()
+        code_location_root,
+        registry or ComponentRegistry.from_entry_point_discovery(),
+        components_folder=components_folder,
     )
 
     all_defs: List[Definitions] = []

python_modules/libraries/dagster-components/dagster_components/core/deployment.py

@@ -1,6 +1,6 @@
 import os
 from pathlib import Path
-from typing import Final, Iterable, Tuple, Type
+from typing import Final, Iterable, Optional, Tuple, Type
 
 import tomli
 from dagster._core.errors import DagsterError
@@ -46,27 +46,38 @@ def _is_code_location_root(path: Path) -> bool:
 
 class CodeLocationProjectContext:
     @classmethod
-    def from_code_location_path(cls, path: Path, component_registry: "ComponentRegistry") -> Self:
+    def from_code_location_path(
+        cls,
+        path: Path,
+        component_registry: "ComponentRegistry",
+        components_folder: Optional[Path] = None,
+    ) -> Self:
         if not _is_code_location_root(path):
             raise DagsterError(
                 f"Path {path} is not a code location root. Must have a pyproject.toml with a [tool.dagster] section."
             )
 
+        name = os.path.basename(path)
+
         return cls(
             root_path=str(path),
-            name=os.path.basename(path),
+            name=name,
             component_registry=component_registry,
+            components_folder=components_folder
+            or path / name / _CODE_LOCATION_COMPONENT_INSTANCES_DIR,
         )
 
     def __init__(
         self,
         root_path: str,
         name: str,
         component_registry: "ComponentRegistry",
+        components_folder: Path,
     ):
         self._root_path = root_path
         self._name = name
         self._component_registry = component_registry
+        self._components_folder = components_folder
 
     @property
     def component_types_root_path(self) -> str:
@@ -99,17 +110,11 @@ def get_component_instance_path(self, name: str) -> str:
 
     @property
     def component_instances_root_path(self) -> str:
-        return os.path.join(self._root_path, self._name, _CODE_LOCATION_COMPONENT_INSTANCES_DIR)
+        return str(self._components_folder)
 
     @property
     def component_instances(self) -> Iterable[str]:
-        return sorted(
-            os.listdir(
-                os.path.join(self._root_path, self._name, _CODE_LOCATION_COMPONENT_INSTANCES_DIR)
-            )
-        )
+        return sorted(os.listdir(self.component_instances_root_path))
 
     def has_component_instance(self, name: str) -> bool:
-        return os.path.exists(
-            os.path.join(self._root_path, self._name, _CODE_LOCATION_COMPONENT_INSTANCES_DIR, name)
-        )
+        return os.path.exists(os.path.join(self.component_instances_root_path, name))