Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve GoodPandas performance #834

Merged
merged 2 commits into from
Oct 7, 2024
Merged

Improve GoodPandas performance #834

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c3f22ac
feat: improve GoodPandas perf
hkad98 Oct 7, 2024
3bee712
test: new cassettes
hkad98 Oct 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 17 additions & 9 deletions gooddata-pandas/gooddata_pandas/data_access.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from gooddata_sdk import (
Attribute,
AttributeFilter,
CatalogWorkspaceContent,
CatalogAttribute,
ExecutionDefinition,
ExecutionResponse,
Filter,
Expand All @@ -16,6 +16,7 @@
ObjId,
TableDimension,
)
from gooddata_sdk.utils import IdObjType
no23reason marked this conversation as resolved.
Show resolved Hide resolved

from gooddata_pandas.utils import (
ColumnsDef,
Expand Down Expand Up @@ -319,27 +320,34 @@ def _extract_for_metrics_only(response: ExecutionResponse, cols: list, col_to_me
return data


def _typed_result(catalog: CatalogWorkspaceContent, attribute: Attribute, result_values: list[Any]) -> list[Any]:
def _find_attribute(attributes: list[CatalogAttribute], id_obj: IdObjType) -> Union[CatalogAttribute, None]:
for attribute in attributes:
if attribute.find_label(id_obj) is not None:
return attribute
return None


def _typed_result(attributes: list[CatalogAttribute], attribute: Attribute, result_values: list[Any]) -> list[Any]:
"""
Internal function to convert result_values to proper data types.
Args:
catalog (CatalogWorkspaceContent): The catalog workspace content.
attributes (list[CatalogAttribute]): The catalog of attributes.
attribute (Attribute): The attribute for which the typed result will be computed.
result_values (list[Any]): A list of raw values.
Returns:
list[Any]: A list of converted values with proper data types.
"""
catalog_attribute = catalog.find_label_attribute(attribute.label)
catalog_attribute = _find_attribute(attributes, attribute.label)
if catalog_attribute is None:
raise ValueError(f"Unable to find attribute {attribute.label} in catalog")
return [_typed_attribute_value(catalog_attribute, value) for value in result_values]


def _extract_from_attributes_and_maybe_metrics(
response: ExecutionResponse,
catalog: CatalogWorkspaceContent,
attributes: list[CatalogAttribute],
cols: list[str],
col_to_attr_idx: dict[str, int],
col_to_metric_idx: dict[str, int],
Expand Down Expand Up @@ -382,12 +390,12 @@ def _extract_from_attributes_and_maybe_metrics(
for idx_name in index:
rs = result.get_all_header_values(attribute_dim, safe_index_to_attr_idx[idx_name])
attribute = index_to_attribute[idx_name]
index[idx_name] += _typed_result(catalog, attribute, rs)
index[idx_name] += _typed_result(attributes, attribute, rs)
for col in cols:
if col in col_to_attr_idx:
rs = result.get_all_header_values(attribute_dim, col_to_attr_idx[col])
attribute = col_to_attribute[col]
data[col] += _typed_result(catalog, attribute, rs)
data[col] += _typed_result(attributes, attribute, rs)
elif col_to_metric_idx[col] < len(result.data):
data[col] += result.data[col_to_metric_idx[col]]
if result.is_complete(attribute_dim):
Expand Down Expand Up @@ -440,10 +448,10 @@ def compute_and_extract(
if not exec_def.has_attributes():
return _extract_for_metrics_only(response, cols, col_to_metric_idx), dict()
else:
catalog = sdk.catalog_workspace_content.get_full_catalog(workspace_id)
attributes = sdk.catalog_workspace_content.get_attributes_catalog(workspace_id, include=["labels", "datasets"])
return _extract_from_attributes_and_maybe_metrics(
response,
catalog,
attributes,
cols,
col_to_attr_idx,
col_to_metric_idx,
Expand Down
14 changes: 7 additions & 7 deletions gooddata-pandas/tests/dataframe/fixtures/dataframe_for_exec_def_bytes_limits_failure.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ interactions:
content-length:
- '1219'
set-cookie:
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:16:43 GMT;
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:18 GMT;
Path=/; HTTPOnly; SameSite=Lax
body:
string:
Expand Down Expand Up @@ -182,10 +182,10 @@ interactions:
name: Order Amount
localIdentifier: dim_1
links:
executionResult: 4e372376a5bef9f8d64b3eec9022efea2e41c7d5:05407e0d701640fb410ed25e354a7e702b55be7bcf8bf949eb4b5fa4776019e2
executionResult: 7ee8ce1d028718cc038ddca8930ed19b19f8aac4:5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3
- request:
method: GET
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/4e372376a5bef9f8d64b3eec9022efea2e41c7d5%3A05407e0d701640fb410ed25e354a7e702b55be7bcf8bf949eb4b5fa4776019e2/metadata
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/7ee8ce1d028718cc038ddca8930ed19b19f8aac4%3A5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3/metadata
body: null
headers:
Accept:
Expand Down Expand Up @@ -252,7 +252,7 @@ interactions:
content-length:
- '2168'
set-cookie:
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:16:43 GMT;
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:18 GMT;
Path=/; HTTPOnly; SameSite=Lax
body:
string:
Expand Down Expand Up @@ -355,7 +355,7 @@ interactions:
name: Order Amount
localIdentifier: dim_1
links:
executionResult: 4e372376a5bef9f8d64b3eec9022efea2e41c7d5:05407e0d701640fb410ed25e354a7e702b55be7bcf8bf949eb4b5fa4776019e2
executionResult: 7ee8ce1d028718cc038ddca8930ed19b19f8aac4:5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3
resultSpec:
dimensions:
- localIdentifier: dim_0
Expand All @@ -372,7 +372,7 @@ interactions:
resultSize: 4237
- request:
method: GET
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/4e372376a5bef9f8d64b3eec9022efea2e41c7d5%3A05407e0d701640fb410ed25e354a7e702b55be7bcf8bf949eb4b5fa4776019e2?offset=0%2C0&limit=100%2C100
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/7ee8ce1d028718cc038ddca8930ed19b19f8aac4%3A5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3?offset=0%2C0&limit=100%2C100
body: null
headers:
Accept:
Expand Down Expand Up @@ -439,7 +439,7 @@ interactions:
content-length:
- '11295'
set-cookie:
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:16:43 GMT;
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:18 GMT;
Path=/; HTTPOnly; SameSite=Lax
body:
string:
Expand Down
14 changes: 7 additions & 7 deletions ...ata-pandas/tests/dataframe/fixtures/dataframe_for_exec_def_dimensions_limits_failure.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ interactions:
content-length:
- '1219'
set-cookie:
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:16:42 GMT;
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:18 GMT;
Path=/; HTTPOnly; SameSite=Lax
body:
string:
Expand Down Expand Up @@ -182,10 +182,10 @@ interactions:
name: Order Amount
localIdentifier: dim_1
links:
executionResult: 4e372376a5bef9f8d64b3eec9022efea2e41c7d5:05407e0d701640fb410ed25e354a7e702b55be7bcf8bf949eb4b5fa4776019e2
executionResult: 7ee8ce1d028718cc038ddca8930ed19b19f8aac4:5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3
- request:
method: GET
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/4e372376a5bef9f8d64b3eec9022efea2e41c7d5%3A05407e0d701640fb410ed25e354a7e702b55be7bcf8bf949eb4b5fa4776019e2/metadata
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/7ee8ce1d028718cc038ddca8930ed19b19f8aac4%3A5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3/metadata
body: null
headers:
Accept:
Expand Down Expand Up @@ -252,7 +252,7 @@ interactions:
content-length:
- '2168'
set-cookie:
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:16:42 GMT;
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:18 GMT;
Path=/; HTTPOnly; SameSite=Lax
body:
string:
Expand Down Expand Up @@ -355,7 +355,7 @@ interactions:
name: Order Amount
localIdentifier: dim_1
links:
executionResult: 4e372376a5bef9f8d64b3eec9022efea2e41c7d5:05407e0d701640fb410ed25e354a7e702b55be7bcf8bf949eb4b5fa4776019e2
executionResult: 7ee8ce1d028718cc038ddca8930ed19b19f8aac4:5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3
resultSpec:
dimensions:
- localIdentifier: dim_0
Expand All @@ -372,7 +372,7 @@ interactions:
resultSize: 4237
- request:
method: GET
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/4e372376a5bef9f8d64b3eec9022efea2e41c7d5%3A05407e0d701640fb410ed25e354a7e702b55be7bcf8bf949eb4b5fa4776019e2?offset=0%2C0&limit=100%2C100
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/7ee8ce1d028718cc038ddca8930ed19b19f8aac4%3A5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3?offset=0%2C0&limit=100%2C100
body: null
headers:
Accept:
Expand Down Expand Up @@ -439,7 +439,7 @@ interactions:
content-length:
- '11295'
set-cookie:
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:16:42 GMT;
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:18 GMT;
Path=/; HTTPOnly; SameSite=Lax
body:
string:
Expand Down
24 changes: 12 additions & 12 deletions gooddata-pandas/tests/dataframe/fixtures/dataframe_for_exec_def_one_dim1.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ interactions:
content-length:
- '1179'
set-cookie:
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:16:48 GMT;
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:22 GMT;
Path=/; HTTPOnly; SameSite=Lax
body:
string:
Expand Down Expand Up @@ -178,10 +178,10 @@ interactions:
name: Order Amount
localIdentifier: dim_0
links:
executionResult: 6d6c03d47ad8bedf8a436f87a7f2638bfe2e0fc8:5e583be017c9937b7fe0f13ba25eb2c731d1b4f9bf129b1384ed5b1b9db86b90
executionResult: c06b00baf3852d8d1dac66f3231bb224be66a113:b65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995
- request:
method: GET
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/6d6c03d47ad8bedf8a436f87a7f2638bfe2e0fc8%3A5e583be017c9937b7fe0f13ba25eb2c731d1b4f9bf129b1384ed5b1b9db86b90/metadata
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/c06b00baf3852d8d1dac66f3231bb224be66a113%3Ab65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995/metadata
body: null
headers:
Accept:
Expand Down Expand Up @@ -248,7 +248,7 @@ interactions:
content-length:
- '2067'
set-cookie:
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:16:48 GMT;
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:22 GMT;
Path=/; HTTPOnly; SameSite=Lax
body:
string:
Expand Down Expand Up @@ -349,7 +349,7 @@ interactions:
name: Order Amount
localIdentifier: dim_0
links:
executionResult: 6d6c03d47ad8bedf8a436f87a7f2638bfe2e0fc8:5e583be017c9937b7fe0f13ba25eb2c731d1b4f9bf129b1384ed5b1b9db86b90
executionResult: c06b00baf3852d8d1dac66f3231bb224be66a113:b65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995
resultSpec:
dimensions:
- localIdentifier: dim_0
Expand All @@ -363,7 +363,7 @@ interactions:
resultSize: 2913
- request:
method: GET
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/6d6c03d47ad8bedf8a436f87a7f2638bfe2e0fc8%3A5e583be017c9937b7fe0f13ba25eb2c731d1b4f9bf129b1384ed5b1b9db86b90?offset=0&limit=500
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/c06b00baf3852d8d1dac66f3231bb224be66a113%3Ab65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995?offset=0&limit=500
body: null
headers:
Accept:
Expand Down Expand Up @@ -430,7 +430,7 @@ interactions:
content-length:
- '98918'
set-cookie:
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:16:48 GMT;
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:22 GMT;
Path=/; HTTPOnly; SameSite=Lax
body:
string:
Expand Down Expand Up @@ -4819,7 +4819,7 @@ interactions:
- 364
- request:
method: GET
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/6d6c03d47ad8bedf8a436f87a7f2638bfe2e0fc8%3A5e583be017c9937b7fe0f13ba25eb2c731d1b4f9bf129b1384ed5b1b9db86b90/metadata
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/c06b00baf3852d8d1dac66f3231bb224be66a113%3Ab65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995/metadata
body: null
headers:
Accept:
Expand Down Expand Up @@ -4886,7 +4886,7 @@ interactions:
content-length:
- '2067'
set-cookie:
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:16:48 GMT;
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:22 GMT;
Path=/; HTTPOnly; SameSite=Lax
body:
string:
Expand Down Expand Up @@ -4987,7 +4987,7 @@ interactions:
name: Order Amount
localIdentifier: dim_0
links:
executionResult: 6d6c03d47ad8bedf8a436f87a7f2638bfe2e0fc8:5e583be017c9937b7fe0f13ba25eb2c731d1b4f9bf129b1384ed5b1b9db86b90
executionResult: c06b00baf3852d8d1dac66f3231bb224be66a113:b65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995
resultSpec:
dimensions:
- localIdentifier: dim_0
Expand All @@ -5001,7 +5001,7 @@ interactions:
resultSize: 2913
- request:
method: GET
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/6d6c03d47ad8bedf8a436f87a7f2638bfe2e0fc8%3A5e583be017c9937b7fe0f13ba25eb2c731d1b4f9bf129b1384ed5b1b9db86b90?offset=0&limit=500
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/c06b00baf3852d8d1dac66f3231bb224be66a113%3Ab65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995?offset=0&limit=500
body: null
headers:
Accept:
Expand Down Expand Up @@ -5068,7 +5068,7 @@ interactions:
content-length:
- '98918'
set-cookie:
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:16:48 GMT;
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:22 GMT;
Path=/; HTTPOnly; SameSite=Lax
body:
string:
Expand Down
Loading
Loading