From b4d2b789cc37dadf2fce9ac99f6e40ada0ca9fd2 Mon Sep 17 00:00:00 2001 From: tubt Date: Fri, 6 Dec 2024 21:27:18 +0700 Subject: [PATCH] feat: POC automation test JIRA: QA-23477 risk: nonprod --- .python-version | 2 +- .../column_sum_amount_by_stage_name.json | 21 +++ ...roduct_filter_department_direct_sales.json | 27 ++++ .../data_response/headline_sum_amount.json | 15 ++ gooddata-sdk/integration_tests/env.py | 9 ++ .../fixtures/ai_questions.json | 14 ++ .../integration_tests/scripts/aiChat.py | 152 ++++++++++++++++++ .../integration_tests/scripts/ai_chat.py | 36 +++++ .../scripts/create_ref_workspace.py | 16 ++ .../scripts/delete_ref_workspace.py | 8 + .../scripts/workspace_manager.py | 73 +++++++++ .../scripts/workspace_manager_test.py | 27 ++++ 12 files changed, 399 insertions(+), 1 deletion(-) create mode 100644 gooddata-sdk/integration_tests/data_response/column_sum_amount_by_stage_name.json create mode 100644 gooddata-sdk/integration_tests/data_response/column_sum_best_case_by_product_filter_department_direct_sales.json create mode 100644 gooddata-sdk/integration_tests/data_response/headline_sum_amount.json create mode 100644 gooddata-sdk/integration_tests/env.py create mode 100644 gooddata-sdk/integration_tests/fixtures/ai_questions.json create mode 100644 gooddata-sdk/integration_tests/scripts/aiChat.py create mode 100644 gooddata-sdk/integration_tests/scripts/ai_chat.py create mode 100644 gooddata-sdk/integration_tests/scripts/create_ref_workspace.py create mode 100644 gooddata-sdk/integration_tests/scripts/delete_ref_workspace.py create mode 100644 gooddata-sdk/integration_tests/scripts/workspace_manager.py create mode 100644 gooddata-sdk/integration_tests/scripts/workspace_manager_test.py diff --git a/.python-version b/.python-version index 455808f8e..0c7d5f5f5 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.12.4 +3.11.4 diff --git a/gooddata-sdk/integration_tests/data_response/column_sum_amount_by_stage_name.json b/gooddata-sdk/integration_tests/data_response/column_sum_amount_by_stage_name.json new file mode 100644 index 000000000..21bfca5f6 --- /dev/null +++ b/gooddata-sdk/integration_tests/data_response/column_sum_amount_by_stage_name.json @@ -0,0 +1,21 @@ +{ + "id": "sum_of_amount_by_stage_name", + "title": "Sum of Amount by Stage Name", + "visualizationType": "COLUMN", + "metrics": [ + { + "id": "f_opportunitysnapshot.f_amount", + "type": "fact", + "title": "Sum of Amount", + "aggFunction": "SUM" + } + ], + "dimensionality": [ + { + "id": "attr.f_stage.stagename", + "type": "attribute", + "title": "Stage Name" + } + ], + "filters": [] +} diff --git a/gooddata-sdk/integration_tests/data_response/column_sum_best_case_by_product_filter_department_direct_sales.json b/gooddata-sdk/integration_tests/data_response/column_sum_best_case_by_product_filter_department_direct_sales.json new file mode 100644 index 000000000..d414c1e43 --- /dev/null +++ b/gooddata-sdk/integration_tests/data_response/column_sum_best_case_by_product_filter_department_direct_sales.json @@ -0,0 +1,27 @@ +{ + "id": "best_case_by_product_for_direct_sales_department", + "title": "Best Case by Product for Direct Sales Department", + "visualizationType": "COLUMN", + "metrics": [ + { + "id": "best_case", + "type": "metric", + "title": "Best Case" + } + ], + "dimensionality": [ + { + "id": "attr.f_product.product", + "type": "attribute", + "title": "Product" + } + ], + "filters": [ + { + "using": "f_owner.department_id", + "include": [ + "Direct Sales" + ] + } + ] +} diff --git a/gooddata-sdk/integration_tests/data_response/headline_sum_amount.json b/gooddata-sdk/integration_tests/data_response/headline_sum_amount.json new file mode 100644 index 000000000..f6cd82ee3 --- /dev/null +++ b/gooddata-sdk/integration_tests/data_response/headline_sum_amount.json @@ -0,0 +1,15 @@ +{ + "id": "total_amount", + "title": "Total Amount", + "visualizationType": "HEADLINE", + "metrics": [ + { + "id": "f_opportunitysnapshot.f_amount", + "type": "fact", + "title": "Sum of Amount", + "aggFunction": "SUM" + } + ], + "dimensionality": [], + "filters": [] + } diff --git a/gooddata-sdk/integration_tests/env.py b/gooddata-sdk/integration_tests/env.py new file mode 100644 index 000000000..cea88bc11 --- /dev/null +++ b/gooddata-sdk/integration_tests/env.py @@ -0,0 +1,9 @@ +# (C) 2024 GoodData Corporation +# env.py +import os + +# Define environment variables +HOST = os.getenv("GOODDATA_HOST", "") +TOKEN = os.getenv("GOODDATA_TOKEN", "") +DATASOURCE_ID = os.getenv("DATASOURCE_ID", "") +WORKSPACE_ID = "" diff --git a/gooddata-sdk/integration_tests/fixtures/ai_questions.json b/gooddata-sdk/integration_tests/fixtures/ai_questions.json new file mode 100644 index 000000000..73bcc90b9 --- /dev/null +++ b/gooddata-sdk/integration_tests/fixtures/ai_questions.json @@ -0,0 +1,14 @@ +[ + { + "question": "generate HEADLINE showing Sum of Amount", + "expected_objects_file": "headline_sum_amount.json" + }, + { + "question": "generate COLUMN chart showing Sum of Amount sliced by Stage Name", + "expected_objects_file": "column_sum_amount_by_stage_name.json" + }, + { + "question": "generate COLUMN chart showing Best Case sliced by Product filtered by Department is 'Direct Sales'", + "expected_objects_file": "column_sum_best_case_by_product_filter_department_direct_sales.json" + } +] diff --git a/gooddata-sdk/integration_tests/scripts/aiChat.py b/gooddata-sdk/integration_tests/scripts/aiChat.py new file mode 100644 index 000000000..5db3b26a4 --- /dev/null +++ b/gooddata-sdk/integration_tests/scripts/aiChat.py @@ -0,0 +1,152 @@ +# (C) 2024 GoodData Corporation +import json +import os +import sys +from pprint import pprint + +import gooddata_api_client +import pytest +from gooddata_api_client.api import smart_functions_api +from gooddata_api_client.model.chat_history_request import ChatHistoryRequest +from gooddata_api_client.model.chat_history_result import ChatHistoryResult +from gooddata_api_client.model.chat_request import ChatRequest + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from env import HOST, TOKEN, WORKSPACE_ID + +EXPECTED_OBJECTS_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data_response") + +QUESTIONS_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "fixtures") + + +@pytest.fixture(scope="module") +def api_client(): + configuration = gooddata_api_client.Configuration(host=HOST) + configuration.access_token = TOKEN + with gooddata_api_client.ApiClient(configuration) as api_client: + yield api_client + + +class GoodDataAiChatApp: + def __init__(self, api_client, workspace_id): + self.api_instance = smart_functions_api.SmartFunctionsApi(api_client) + self.workspace_id = workspace_id + + async def ask_question(self, question: str): + chat_request = ChatRequest(question=question) + return self.api_instance.ai_chat(self.workspace_id, chat_request) + + async def chat_history(self, chat_history_interaction_id: int, user_feedback: str): + chat_history_request = ChatHistoryRequest( + chat_history_interaction_id=chat_history_interaction_id, + user_feedback=user_feedback, + ) + return self.api_instance.ai_chat_history(self.workspace_id, chat_history_request) + + +def set_authorization_header(api_client, token): + api_client.default_headers["Authorization"] = f"Bearer {token}" + + +def snake_to_camel(snake_str): + """Convert snake_case to camelCase.""" + components = snake_str.split("_") + return components[0] + "".join(x.title() for x in components[1:]) + + +def normalize_metrics(metrics, exclude_keys=None): + """Normalize keys in the metrics list, excluding specified keys.""" + if exclude_keys is None: + exclude_keys = [] + normalized_metrics = [] + for metric in metrics: + if isinstance(metric, dict): + normalized_metric = {} + for key, value in metric.items(): + if key in exclude_keys: + continue + else: + new_key = snake_to_camel(key) + normalized_metric[new_key] = value + normalized_metrics.append(normalized_metric) + return normalized_metrics + + +def handle_api_response_ai_chat(api_response, expected_objects): + response_dict = api_response.to_dict() + + # Normalize the keys in the actual and expected metrics, excluding 'title' + actual_metrics = normalize_metrics( + response_dict["created_visualizations"]["objects"][0]["metrics"], exclude_keys=["title"] + ) + expected_metrics = normalize_metrics(expected_objects["metrics"], exclude_keys=["title"]) + + assert actual_metrics == expected_metrics, "Metrics do not match" + assert ( + response_dict["created_visualizations"]["objects"][0]["visualization_type"] + == expected_objects["visualizationType"] + ), "Visualization type does not match" + assert ( + response_dict["created_visualizations"]["objects"][0]["dimensionality"] == expected_objects["dimensionality"] + ), "Dimensionality does not match" + assert ( + response_dict["created_visualizations"]["objects"][0]["filters"] == expected_objects["filters"] + ), "Filters do not match" + + +def getChatHistoryInteractionId(api_response): + chat_history_interaction_id = api_response.chat_history_interaction_id + print(f"Chat history interaction id: {chat_history_interaction_id}") + return chat_history_interaction_id + + +@pytest.fixture(scope="module") +def app(api_client): + app = GoodDataAiChatApp(api_client, WORKSPACE_ID) + set_authorization_header(api_client, TOKEN) + return app + + +def load_expected_objects(fixture_folder, filename): + with open(os.path.join(fixture_folder, filename)) as file: + return json.load(file) + + +questions_list = load_expected_objects(QUESTIONS_DIR, "ai_questions.json") + + +@pytest.mark.parametrize( + "question,expected_objects_file", + [(item["question"], item["expected_objects_file"]) for item in questions_list], + ids=[item["question"] for item in questions_list], +) +@pytest.mark.asyncio +async def test_ai_chat(app, question, expected_objects_file): + expected_objects = load_expected_objects(EXPECTED_OBJECTS_DIR, expected_objects_file) + try: + api_response = await app.ask_question(question) + handle_api_response_ai_chat(api_response, expected_objects) + except gooddata_api_client.ApiException as e: + pytest.fail(f"Exception when calling SmartFunctionsApi->ai_chat: {e}\n") + except Exception as e: + pytest.fail(f"An unexpected error occurred: {e}\n") + + +@pytest.mark.asyncio +async def test_ai_chat_history(app): + try: + api_response = await app.chat_history(1260, "POSITIVE") + + assert isinstance(api_response, ChatHistoryResult), "Response is not of type ChatHistoryResult" + pprint(api_response.to_dict()) + except gooddata_api_client.ApiException as e: + print(f"Exception when calling SmartFunctionsApi->ai_chat_history: {e}") + pytest.fail(f"Exception when calling SmartFunctionsApi->ai_chat_history: {e}\n") + except Exception as e: + print(f"An unexpected error occurred: {e}") + pytest.fail(f"An unexpected error occurred: {e}\n") + + +if __name__ == "__main__": + pytest.main() diff --git a/gooddata-sdk/integration_tests/scripts/ai_chat.py b/gooddata-sdk/integration_tests/scripts/ai_chat.py new file mode 100644 index 000000000..b16a2c783 --- /dev/null +++ b/gooddata-sdk/integration_tests/scripts/ai_chat.py @@ -0,0 +1,36 @@ +# (C) 2024 GoodData Corporation +import os +import sys + +import pytest + +# Add the root directory to sys.path +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from env import HOST, TOKEN, WORKSPACE_ID +from gooddata_sdk import GoodDataSdk + + +@pytest.fixture +def test_config(): + return {"host": HOST, "token": TOKEN, "workspace_id": WORKSPACE_ID} + + +questions = [ + "What is the number of Accounts?", + "What is the total of Amount?", +] + + +@pytest.mark.parametrize("question", questions) +def test_ask_ai(test_config, question): + sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"]) + workspace_id = test_config["workspace_id"] + chat_ai_res = sdk.compute.ai_chat(workspace_id, question=question) + + print(f"Chat AI response: {chat_ai_res}") + assert chat_ai_res is not None, "Response should not be None" + + +if __name__ == "__main__": + pytest.main() diff --git a/gooddata-sdk/integration_tests/scripts/create_ref_workspace.py b/gooddata-sdk/integration_tests/scripts/create_ref_workspace.py new file mode 100644 index 000000000..dc07c6e7c --- /dev/null +++ b/gooddata-sdk/integration_tests/scripts/create_ref_workspace.py @@ -0,0 +1,16 @@ +# (C) 2024 GoodData Corporation +from env import DATASOURCE_ID, HOST, TOKEN, WORKSPACE_ID +from workspace_manager import createWorkspace, getDataSource, update_env_file + +if __name__ == "__main__": + test_config = {"host": HOST, "token": TOKEN} + + if WORKSPACE_ID: + print(f"Workspace ID '{WORKSPACE_ID}' already exists. Skipping workspace creation.") + else: + workspace_id = createWorkspace(test_config) + dataSource = getDataSource(DATASOURCE_ID, test_config) + if workspace_id: + update_env_file(workspace_id) + else: + print("Failed to create workspace.") diff --git a/gooddata-sdk/integration_tests/scripts/delete_ref_workspace.py b/gooddata-sdk/integration_tests/scripts/delete_ref_workspace.py new file mode 100644 index 000000000..a4a97b9c7 --- /dev/null +++ b/gooddata-sdk/integration_tests/scripts/delete_ref_workspace.py @@ -0,0 +1,8 @@ +# (C) 2024 GoodData Corporation +from env import HOST, TOKEN +from workspace_manager import deleteWorkspace + +if __name__ == "__main__": + test_config = {"host": HOST, "token": TOKEN} + + deleteWorkspace(test_config) diff --git a/gooddata-sdk/integration_tests/scripts/workspace_manager.py b/gooddata-sdk/integration_tests/scripts/workspace_manager.py new file mode 100644 index 000000000..1d6a05555 --- /dev/null +++ b/gooddata-sdk/integration_tests/scripts/workspace_manager.py @@ -0,0 +1,73 @@ +# (C) 2024 GoodData Corporation +import os +import sys +import time +import uuid + +from gooddata_sdk import CatalogWorkspace, GoodDataSdk + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +try: + from env import WORKSPACE_ID +except ImportError: + WORKSPACE_ID = None + + +def createWorkspace(test_config): + sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"]) + + workspace_id = uuid.uuid4().hex + timestamp = int(time.time()) + workspace_name = f"pysdk_test_{timestamp}" + + workspace = CatalogWorkspace(workspace_id, workspace_name) + try: + sdk.catalog_workspace.create_or_update(workspace) + workspace_o = sdk.catalog_workspace.get_workspace(workspace_id) + assert workspace_o == workspace + + print(f"Workspace '{workspace_name}' with ID '{workspace_id}' created successfully.") + return workspace_id + except Exception as e: + print(f"An error occurred while creating the workspace: {e}") + return None + + +def deleteWorkspace(test_config): + sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"]) + try: + workspaces = sdk.catalog_workspace.list_workspaces() + for workspace in workspaces: + if workspace.name.startswith("pysdk_test_"): + sdk.catalog_workspace.delete_workspace(workspace.id) + print(f"Workspace '{workspace.name}' with ID '{workspace.id}' deleted successfully.") + remove_env_file() + except Exception as e: + print(f"An error occurred while deleting workspaces: {e}") + + +def update_env_file(workspace_id): + with open("env.py", "a") as f: + f.write(f'\nWORKSPACE_ID = "{workspace_id}"\n') + + +def remove_env_file(): + try: + with open("env.py") as f: # Default mode is 'r' + lines = f.readlines() + with open("env.py", "w") as f: + for line in lines: + if "WORKSPACE_ID" not in line: + f.write(line) + print("Removed WORKSPACE_ID from env.py") + except Exception as e: + print(f"An error occurred while removing WORKSPACE_ID from env.py: {e}") + + +def getDataSource(data_source_id, test_config): + sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"]) + data_source = sdk.catalog_data_source.get_data_source(data_source_id) + data_source_schema = data_source.schema + print(f"Data source schema: {data_source_schema}") + return data_source_schema diff --git a/gooddata-sdk/integration_tests/scripts/workspace_manager_test.py b/gooddata-sdk/integration_tests/scripts/workspace_manager_test.py new file mode 100644 index 000000000..1e80753e1 --- /dev/null +++ b/gooddata-sdk/integration_tests/scripts/workspace_manager_test.py @@ -0,0 +1,27 @@ +# (C) 2024 GoodData Corporation +# import pytest +# from workspace_manager import create_workspace, delete_workspace, get_data_source, update_env_file +# from env import DATASOURCE_ID, HOST, TOKEN +# @pytest.fixture +# def test_config(): +# return { +# 'host': HOST, +# 'token': TOKEN +# } + +# def test_create_workspace(test_config): +# workspace_id = create_workspace(test_config) +# assert workspace_id is not None, "Workspace creation failed" +# update_env_file(workspace_id) + +# # def test_delete_workspace(test_config): +# # delete_workspace(test_config) +# # # Assuming the function prints the deletion message, we can check the output +# # # Here we assume that the function works correctly if no exception is raised + +# def test_get_data_source(test_config): +# schema = get_data_source(DATASOURCE_ID, test_config) +# assert schema is not None, "Failed to get data source schema" + +# if __name__ == "__main__": +# pytest.main()