Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[bot] Merge master/ea408475 into rel/dev #661

Merged
merged 2 commits into from
Apr 23, 2024
Merged

[bot] Merge master/ea408475 into rel/dev #661

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
77d669c
fix(gooddata-pandas): Store primary labels mapping in DF metadata
torchbear Apr 23, 2024
ea40847
Merge pull request #660 from gooddata/rha/cq-345/collect-primary-labe…
Apr 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 53 additions & 23 deletions gooddata-pandas/gooddata_pandas/result_convertor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,12 +200,16 @@ class DataFrameMetadata:

row_totals_indexes: List[List[int]]
execution_response: BareExecutionResponse
primary_labels_from_index: Dict[int, Dict[str, str]]
primary_labels_from_columns: Dict[int, Dict[str, str]]

@classmethod
def from_data(
cls,
headers: Tuple[_DataHeaders, Optional[_DataHeaders]],
execution_response: BareExecutionResponse,
primary_labels_from_index: Dict[int, Dict[str, str]],
primary_labels_from_columns: Dict[int, Dict[str, str]],
) -> "DataFrameMetadata":
"""This method constructs a DataFrameMetadata object from data headers and an execution response.

Expand All @@ -219,6 +223,8 @@ def from_data(
return cls(
row_totals_indexes=row_totals_indexes,
execution_response=execution_response,
primary_labels_from_index=primary_labels_from_index,
primary_labels_from_columns=primary_labels_from_columns,
)


Expand Down Expand Up @@ -304,6 +310,7 @@ def _read_complete_execution_result(
def _create_header_mapper(
response: BareExecutionResponse,
dim: int,
primary_attribute_labels_mapping: Dict[int, Dict[str, str]],
label_overrides: Optional[LabelOverrides] = None,
use_local_ids_in_headers: bool = False,
use_primary_labels_in_attributes: bool = False,
Expand All @@ -315,6 +322,8 @@ def _create_header_mapper(
Args:
response (BareExecutionResponse): Response structure to gather dimension header details.
dim (int): Dimension id.
primary_attribute_labels_mapping (Dict[int, Dict[str, str]]): Dict to be filled by mapping of primary labels to
custom labels per level identified by integer.
label_overrides (Optional[LabelOverrides]): Label overrides. Defaults to None.
use_local_ids_in_headers (bool): Use local identifiers of header attributes and metrics. Optional.
Defaults to False.
Expand All @@ -336,10 +345,17 @@ def _mapper(header: Any, header_idx: Optional[int]) -> Optional[str]:
pass
elif "attributeHeader" in header:
if "labelValue" in header["attributeHeader"]:
label_value = header["attributeHeader"]["labelValue"]
primary_label_value = header["attributeHeader"]["primaryLabelValue"]
if use_primary_labels_in_attributes:
label = header["attributeHeader"]["primaryLabelValue"]
label = primary_label_value
else:
label = header["attributeHeader"]["labelValue"]
label = label_value
if header_idx is not None:
if header_idx in primary_attribute_labels_mapping:
primary_attribute_labels_mapping[header_idx][primary_label_value] = label_value
else:
primary_attribute_labels_mapping[header_idx] = {primary_label_value: label_value}
# explicitly handle '(empty value)' if it's None otherwise it's not recognizable in final MultiIndex
# backend represents ^^^ by "" (datasource value is "") or None (datasource value is NULL) therefore
# if both representation are used it's necessary to set label to unique header label (space) to avoid
Expand Down Expand Up @@ -382,7 +398,7 @@ def _headers_to_index(
label_overrides: LabelOverrides,
use_local_ids_in_headers: bool = False,
use_primary_labels_in_attributes: bool = False,
) -> Optional[pandas.Index]:
) -> Tuple[Optional[pandas.Index], Dict[int, Dict[str, str]]]:
"""Converts headers to a pandas MultiIndex.

This function converts the headers present in the response to a pandas MultiIndex (can be used in pandas dataframes)
Expand All @@ -398,17 +414,22 @@ def _headers_to_index(
Defaults to False.

Returns:
Optional[pandas.Index]: A pandas MultiIndex object created from the headers, or None if the headers are empty.
Tuple[Optional[pandas.Index], Dict[int, Dict[str, str]]: A pandas MultiIndex object created from the headers
with primary attribute labels mapping as Dict, or None with empty Dict if the headers are empty.
"""
# dict of primary labels and it's custom labels for attributes per level as key
primary_attribute_labels_mapping: Dict[int, Dict[str, str]] = {}

if len(response.dimensions) <= dim_idx or not len(response.dimensions[dim_idx]["headers"]):
return None
return None, primary_attribute_labels_mapping

mapper = _create_header_mapper(
response=response,
dim=dim_idx,
label_overrides=label_overrides,
use_local_ids_in_headers=use_local_ids_in_headers,
use_primary_labels_in_attributes=use_primary_labels_in_attributes,
primary_attribute_labels_mapping=primary_attribute_labels_mapping,
)

return pandas.MultiIndex.from_arrays(
Expand All @@ -417,7 +438,7 @@ def _headers_to_index(
for header_idx, header_group in enumerate(cast(_DataHeaders, headers[dim_idx]))
],
names=[mapper(dim_header, None) for dim_header in (response.dimensions[dim_idx]["headers"])],
)
), primary_attribute_labels_mapping


def _merge_grand_totals_into_data(extract: _DataWithHeaders) -> Union[_DataArray, List[_DataArray]]:
Expand Down Expand Up @@ -507,24 +528,33 @@ def convert_execution_response_to_dataframe(
full_data = _merge_grand_totals_into_data(extract)
full_headers = _merge_grand_total_headers_into_headers(extract)

index, primary_labels_from_index = _headers_to_index(
dim_idx=0,
headers=full_headers,
response=execution_response,
label_overrides=label_overrides,
use_local_ids_in_headers=use_local_ids_in_headers,
use_primary_labels_in_attributes=use_primary_labels_in_attributes,
)

columns, primary_labels_from_columns = _headers_to_index(
dim_idx=1,
headers=full_headers,
response=execution_response,
label_overrides=label_overrides,
use_local_ids_in_headers=use_local_ids_in_headers,
use_primary_labels_in_attributes=use_primary_labels_in_attributes,
)

df = pandas.DataFrame(
data=full_data,
index=_headers_to_index(
dim_idx=0,
headers=full_headers,
response=execution_response,
label_overrides=label_overrides,
use_local_ids_in_headers=use_local_ids_in_headers,
use_primary_labels_in_attributes=use_primary_labels_in_attributes,
),
columns=_headers_to_index(
dim_idx=1,
headers=full_headers,
response=execution_response,
label_overrides=label_overrides,
use_local_ids_in_headers=use_local_ids_in_headers,
use_primary_labels_in_attributes=use_primary_labels_in_attributes,
),
index=index,
columns=columns,
)

return df, DataFrameMetadata.from_data(headers=full_headers, execution_response=execution_response)
return df, DataFrameMetadata.from_data(
headers=full_headers,
execution_response=execution_response,
primary_labels_from_index=primary_labels_from_index,
primary_labels_from_columns=primary_labels_from_columns,
)
Loading