feat: support specifying flow file for test --ui (#3287)

# Description

Please add an informative description that covers that changes made by
the pull request and link all relevant issues.

# All Promptflow Contribution checklist:
- [x] **The pull request does not introduce [breaking changes].**
- [ ] **CHANGELOG is updated for new features, bug fixes or other
significant changes.**
- [ ] **I have read the [contribution guidelines](../CONTRIBUTING.md).**
- [ ] **Create an issue and link to the pull request to get dedicated
review from promptflow team. Learn more: [suggested
workflow](../CONTRIBUTING.md#suggested-workflow).**

## General Guidelines and Best Practices
- [ ] Title of the pull request is clear and informative.
- [ ] There are a small number of commits, each of which have an
informative message. This means that previously merged commits do not
appear in the history of the PR. For more information on cleaning up the
commits in your PR, [see this
page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md).

### Testing Guidelines
- [ ] Pull request includes test coverage for the included changes.

src/promptflow-core/promptflow/core/_serving/app.py

@@ -36,6 +36,7 @@ def create_app(**kwargs):
         from promptflow.core._serving.v2.app import PromptFlowServingAppV2
 
         logger = LoggerFactory.get_logger("pfserving-app-v2", target_stdout=True)
+        # TODO: support specify flow file path in fastapi app
         app = PromptFlowServingAppV2(docs_url=None, redoc_url=None, logger=logger, **kwargs)  # type: ignore
         # enable auto-instrumentation if customer installed opentelemetry-instrumentation-fastapi
         try:

src/promptflow-core/promptflow/core/_serving/app_base.py

@@ -5,6 +5,7 @@
 import mimetypes
 import os
 from abc import ABC, abstractmethod
+from pathlib import Path
 from typing import Dict
 
 from promptflow._utils.flow_utils import resolve_flow_path
@@ -38,8 +39,16 @@ def init_app(self, **kwargs):
         # parse promptflow project path
         self.project_path = self.extension.get_flow_project_path()
         logger.info(f"Project path: {self.project_path}")
-        flow_dir, flow_file_name = resolve_flow_path(self.project_path, allow_prompty_dir=True)
-        self.flow = init_executable(flow_path=flow_dir / flow_file_name)
+
+        flow_file_path = kwargs.get("flow_file_path", None)
+        if flow_file_path:
+            self.flow_file_path = Path(flow_file_path)
+        else:
+            flow_dir, flow_file_name = resolve_flow_path(self.project_path, allow_prompty_dir=True)
+            # project path is also the current working directory
+            self.flow_file_path = flow_dir / flow_file_name
+
+        self.flow = init_executable(flow_path=self.flow_file_path, working_dir=Path(self.project_path))
 
         # enable environment_variables
         environment_variables = kwargs.get("environment_variables", {})
@@ -97,9 +106,8 @@ def init_invoker_if_not_exist(self):
         if self.flow_invoker:
             return
         self.logger.info("Promptflow executor starts initializing...")
-        flow_dir, flow_file_name = resolve_flow_path(self.project_path, allow_prompty_dir=True)
         self.flow_invoker = AsyncFlowInvoker(
-            flow=Flow.load(source=flow_dir / flow_file_name),
+            flow=Flow.load(source=self.flow_file_path),
             connection_provider=self.connection_provider,
             streaming=self.streaming_response_required,
             raise_ex=False,

src/promptflow-devkit/promptflow/_cli/_pf/_flow.py

@@ -41,7 +41,6 @@
 from promptflow._sdk._constants import DEFAULT_SERVE_ENGINE, PROMPT_FLOW_DIR_NAME
 from promptflow._sdk._pf_client import PFClient
 from promptflow._sdk._utilities.chat_utils import start_chat_ui_service_monitor
-from promptflow._sdk._utilities.general_utils import generate_yaml_entry_without_delete
 from promptflow._sdk._utilities.serve_utils import find_available_port, start_flow_service
 from promptflow._utils.flow_utils import is_flex_flow
 from promptflow._utils.logger_utils import get_cli_sdk_logger
@@ -525,11 +524,9 @@ def _test_flow_multi_modal(args, pf_client, environment_variables):
 
         pfs_port = _invoke_pf_svc()
         serve_app_port = args.port or find_available_port()
-        flow = generate_yaml_entry_without_delete(entry=args.flow)
-        # flex flow without yaml file doesn't support /eval in chat window
-        enable_internal_features = Configuration.get_instance().is_internal_features_enabled() and flow == args.flow
+        enable_internal_features = Configuration.get_instance().is_internal_features_enabled()
         start_chat_ui_service_monitor(
-            flow=flow,
+            flow=args.flow,
             serve_app_port=serve_app_port,
             pfs_port=pfs_port,
             url_params=list_of_dict_to_dict(args.url_params),

src/promptflow-devkit/promptflow/_sdk/_utilities/chat_utils.py

@@ -1,15 +1,20 @@
 import json
-import webbrowser
+import shutil
 from pathlib import Path
 from typing import Any, Dict
 from urllib.parse import urlencode, urlunparse
 
 from promptflow._constants import FlowLanguage
 from promptflow._sdk._constants import DEFAULT_ENCODING, PROMPT_FLOW_DIR_NAME, UX_INPUTS_INIT_KEY, UX_INPUTS_JSON
 from promptflow._sdk._service.utils.utils import encrypt_flow_path
-from promptflow._sdk._utilities.general_utils import resolve_flow_language
+from promptflow._sdk._utilities.general_utils import (
+    _get_additional_includes,
+    generate_yaml_entry_without_delete,
+    resolve_flow_language,
+)
 from promptflow._sdk._utilities.monitor_utils import (
     DirectoryModificationMonitorTarget,
+    FileModificationMonitorTarget,
     JsonContentMonitorTarget,
     Monitor,
 )
@@ -37,49 +42,57 @@ def construct_chat_page_url(flow_path, port, url_params):
 def _try_restart_service(
     *,
     last_result: ServeAppHelper,
-    flow_file_name: str,
+    flow_file_path: Path,
     flow_dir: Path,
     serve_app_port: int,
     ux_input_path: Path,
     environment_variables: Dict[str, str],
+    chat_page_url: str,
+    skip_open_browser: bool,
 ):
     if last_result is not None:
+        last_helper, skip_open_browser = last_result
         print_log("Changes detected, stopping current serve app...")
-        last_result.terminate()
+        last_helper.terminate()
 
     # init must be always loaded from ux_inputs.json
     if not ux_input_path.is_file():
         init = {}
     else:
         ux_inputs = json.loads(ux_input_path.read_text(encoding=DEFAULT_ENCODING))
-        init = ux_inputs.get(UX_INPUTS_INIT_KEY, {}).get(flow_file_name, {})
+        init = ux_inputs.get(UX_INPUTS_INIT_KEY, {}).get(flow_file_path.name, {})
 
-    language = resolve_flow_language(flow_path=flow_file_name, working_dir=flow_dir)
+    language = resolve_flow_language(flow_path=flow_file_path, working_dir=flow_dir)
     if language == FlowLanguage.Python:
         # additional includes will always be called by the helper.
         # This is expected as user will change files in original locations only
         helper = PythonServeAppHelper(
-            flow_file_name=flow_file_name,
+            flow_file_path=flow_file_path,
             flow_dir=flow_dir,
             init=init,
             port=serve_app_port,
             environment_variables=environment_variables,
+            chat_page_url=chat_page_url,
         )
     else:
         helper = CSharpServeAppHelper(
-            flow_file_name=flow_file_name,
+            flow_file_path=flow_file_path,
             flow_dir=flow_dir,
             init=init,
             port=serve_app_port,
             environment_variables=environment_variables,
+            chat_page_url=chat_page_url,
         )
 
     print_log("Starting serve app...")
     try:
-        helper.start()
+        helper.start(skip_open_browser=skip_open_browser)
+        # only open on first successful start
+        skip_open_browser = True
     except Exception:
         print_log("Failed to start serve app, please check the error message above.")
-    return helper
+    finally:
+        return helper, skip_open_browser
 
 
 def update_init_in_ux_inputs(*, ux_input_path: Path, flow_file_name: str, init: Dict[str, Any]):
@@ -119,7 +132,7 @@ def touch_local_pfs():
 
 
 def start_chat_ui_service_monitor(
-    flow,
+    flow: str,
     *,
     serve_app_port: str,
     pfs_port: str,
@@ -129,42 +142,82 @@ def start_chat_ui_service_monitor(
     skip_open_browser: bool = False,
     environment_variables: Dict[str, str] = None,
 ):
-    flow_dir, flow_file_name = resolve_flow_path(flow, allow_prompty_dir=True)
+    # if flow is an entry, generate yaml entry without delete; if flow is a path, use it directly
+    flow_file_path = generate_yaml_entry_without_delete(entry=flow)
+    use_entry_as_flow = flow_file_path != flow
+    if use_entry_as_flow:
+        flow_dir = Path(".")
+        flow_file_name = flow_file_path.name
+        # when using entry as flow, chat UI will now access ux_inputs.json and images in temp directory
+        ux_input_path = flow_file_path.parent / PROMPT_FLOW_DIR_NAME / UX_INPUTS_JSON
+
+        # so we need to copy the original ux_inputs.json to temp directory
+        origin_prompt_flow_dir = flow_dir / PROMPT_FLOW_DIR_NAME
+        if origin_prompt_flow_dir.is_dir():
+            # TODO: skip files that no need to copy, like logs for flow run
+            shutil.copytree(origin_prompt_flow_dir, flow_file_path.parent / PROMPT_FLOW_DIR_NAME)
+    else:
+        flow_dir, flow_file_name = resolve_flow_path(flow, allow_prompty_dir=True)
+        flow_file_path = flow_dir / flow_file_name
+        ux_input_path = flow_dir / PROMPT_FLOW_DIR_NAME / UX_INPUTS_JSON
+        origin_prompt_flow_dir = None
 
-    ux_input_path = flow_dir / PROMPT_FLOW_DIR_NAME / UX_INPUTS_JSON
     update_init_in_ux_inputs(ux_input_path=ux_input_path, flow_file_name=flow_file_name, init=init)
 
     # show url for chat UI
     url_params["serve_app_port"] = serve_app_port
     if "enable_internal_features" not in url_params:
-        url_params["enable_internal_features"] = "true" if enable_internal_features else "false"
+        # /eval is not supported in chat window when using entry as a flow for now
+        url_params["enable_internal_features"] = (
+            "true" if enable_internal_features and not use_entry_as_flow else "false"
+        )
     chat_page_url = construct_chat_page_url(
-        str(flow_dir / flow_file_name),
+        # Chat UI now doesn't support as_posix in windows
+        str(flow_file_path),
         pfs_port,
         url_params=url_params,
     )
-    print_log(f"You can begin chat flow on {chat_page_url}")
-    if not skip_open_browser:
-        webbrowser.open(chat_page_url)
+
+    targets = [
+        DirectoryModificationMonitorTarget(
+            target=flow_dir,
+            relative_root_ignores=[PROMPT_FLOW_DIR_NAME, "__pycache__"],
+        ),
+        JsonContentMonitorTarget(
+            target=ux_input_path,
+            node_path=[UX_INPUTS_INIT_KEY, flow_file_name],
+        ),
+    ]
+
+    for additional_includes in _get_additional_includes(flow_file_path):
+        target = Path(additional_includes)
+        if target.is_file():
+            targets.append(
+                FileModificationMonitorTarget(
+                    target=target,
+                )
+            )
+        elif target.is_dir():
+            targets.append(
+                DirectoryModificationMonitorTarget(
+                    target=target,
+                    relative_root_ignores=["__pycache__"],
+                )
+            )
+
+    print_log(f"Chat page URL will be available after service is started: {chat_page_url}")
 
     monitor = Monitor(
-        targets=[
-            DirectoryModificationMonitorTarget(
-                target=flow_dir,
-                relative_root_ignores=[PROMPT_FLOW_DIR_NAME, "__pycache__"],
-            ),
-            JsonContentMonitorTarget(
-                target=ux_input_path,
-                node_path=[UX_INPUTS_INIT_KEY, flow_file_name],
-            ),
-        ],
+        targets=targets,
         target_callback=_try_restart_service,
         target_callback_kwargs={
-            "flow_file_name": flow_file_name,
+            "flow_file_path": flow_file_path,
             "flow_dir": flow_dir,
             "serve_app_port": int(serve_app_port),
             "ux_input_path": ux_input_path,
             "environment_variables": environment_variables,
+            "chat_page_url": chat_page_url,
+            "skip_open_browser": skip_open_browser,
         },
         inject_last_callback_result=True,
         extra_logic_in_loop=touch_local_pfs,
@@ -174,7 +227,10 @@ def start_chat_ui_service_monitor(
         monitor.start_monitor()
     except KeyboardInterrupt:
         print_log("Stopping monitor and attached serve app...")
-        serve_app_helper = monitor.last_callback_result
-        if serve_app_helper is not None:
+        if monitor.last_callback_result is not None:
+            serve_app_helper, _ = monitor.last_callback_result
             serve_app_helper.terminate()
         print_log("Stopped monitor and attached serve app.")
+    finally:
+        if use_entry_as_flow:
+            shutil.copytree(flow_file_path.parent / PROMPT_FLOW_DIR_NAME, origin_prompt_flow_dir, dirs_exist_ok=True)

src/promptflow-devkit/promptflow/_sdk/_utilities/general_utils.py

@@ -981,7 +981,7 @@ def resolve_entry_and_code(entry: Union[str, PathLike, Callable], code: Path = N
     return entry, code
 
 
-def create_flex_flow_yaml_in_target(entry: Union[str, PathLike, Callable], target_dir: str, code: Path = None):
+def create_flex_flow_yaml_in_target(entry: Union[str, PathLike, Callable], target_dir: str, code: Path = None) -> Path:
     """
     Generate a flex flow yaml in target folder. The code field in the yaml points to the original flex yaml flow folder.
     """