-
Notifications
You must be signed in to change notification settings - Fork 1.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[ui] Asset view support for column lineage metadata (#20525)
## Summary & Motivation This PR adds a new "Column" selector to the Asset > Lineage page for assets that emit `lineage` materialization event metadata. Choosing a column switches you to a column-oriented version of the graph, which supports the same upstream/downstream scoping + navigation as the normal lineage view, but shows column-level arrows and info. This PR makes a few changes to our asset layout system in order to support this new DAG styling: - The asset layout engine takes a wider range of it's hardcoded constants as configuration (previously just `direction`), and you can override Dagre layout settings. This allows us to run asset layout (with the worker, indexdb caching, etc), but with smaller `column` nodes. The options were already part of the cache keys, so this all works pretty smoothly! - I found an open PR on Dagre that fixes the issue with asset groups overlapping occasionally because the layout engine could not account for the presence of the "title bar" we render on them! I added the adjustments to our existing `dagre.patch` file. This is important in this PR because the nodes are the columns and the groups are the assets, so they overlap badly without this fix. ## How I Tested These Changes <img width="1728" alt="image" src="https://github.com/dagster-io/dagster/assets/1037212/0f5aebfd-cb5b-436e-b344-23b6a8f85870"> <img width="1563" alt="image" src="https://github.com/dagster-io/dagster/assets/1037212/4eccf376-4835-42b1-ad2c-590cb3917b1a"> <img width="1205" alt="image" src="https://github.com/dagster-io/dagster/assets/1037212/16f659a9-cb90-489b-b0ce-03a09a4e1a9c"> --------- Co-authored-by: bengotow <[email protected]>
- Loading branch information
Showing
27 changed files
with
1,056 additions
and
179 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
6 changes: 6 additions & 0 deletions
6
js_modules/dagster-ui/packages/ui-components/src/icon-svgs/column_lineage.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
115 changes: 115 additions & 0 deletions
115
js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetColumnsNode.tsx
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,115 @@ | ||
| import {Box, Caption, Colors, Icon, StyledTag, Tooltip} from '@dagster-io/ui-components'; | ||
| import {Link} from 'react-router-dom'; | ||
| import styled from 'styled-components'; | ||
|
|
||
| import { | ||
| AssetInsetForHoverEffect, | ||
| AssetNameRow, | ||
| AssetNodeBox, | ||
| AssetNodeContainer, | ||
| } from './AssetNode'; | ||
| import {AssetNodeFragment} from './types/AssetNode.types'; | ||
| import {Timestamp} from '../app/time/Timestamp'; | ||
| import {assetDetailsPathForKey} from '../assets/assetDetailsPathForKey'; | ||
| import {AssetColumnLineageLocalColumn} from '../assets/lineage/useColumnLineageDataForAssets'; | ||
| import {AssetComputeKindTag} from '../graph/OpTags'; | ||
| import {AssetKeyInput} from '../graphql/types'; | ||
| import {iconForColumnType} from '../metadata/TableSchema'; | ||
| import {Description} from '../pipelines/Description'; | ||
|
|
||
| export const AssetColumnsGroupNode = ({ | ||
| selected, | ||
| definition, | ||
| height, | ||
| asOf, | ||
| }: { | ||
| selected: boolean; | ||
| definition: AssetNodeFragment; | ||
| asOf: string | undefined; | ||
| height: number; | ||
| }) => { | ||
| return ( | ||
| <AssetInsetForHoverEffect style={{marginLeft: 12, marginRight: 12}}> | ||
| <AssetNodeContainer $selected={selected}> | ||
| <div style={{minHeight: 24}} /> | ||
| <AssetNodeBox $selected={selected} $isSource={definition.isSource} $noScale> | ||
| <AssetNameRow definition={definition} /> | ||
| <Box style={{height: height - 60}} flex={{direction: 'column'}}></Box> | ||
| <Box border="top" padding={{horizontal: 8, vertical: 2}} style={{minHeight: 22}}> | ||
| {asOf ? ( | ||
| <Caption color={Colors.textLighter()}> | ||
| <Timestamp timestamp={{ms: Number(asOf)}} /> | ||
| </Caption> | ||
| ) : undefined} | ||
| </Box> | ||
| </AssetNodeBox> | ||
| <AssetComputeKindTag definition={definition} style={{right: 10, paddingTop: 7}} /> | ||
| </AssetNodeContainer> | ||
| </AssetInsetForHoverEffect> | ||
| ); | ||
| }; | ||
|
|
||
| export const AssetColumnNode = ({ | ||
| assetKey, | ||
| column, | ||
| blueBackground, | ||
| }: { | ||
| assetKey: AssetKeyInput; | ||
| column: AssetColumnLineageLocalColumn; | ||
| blueBackground: boolean; | ||
| }) => { | ||
| const icon = iconForColumnType(column.type ?? ''); | ||
|
|
||
| return ( | ||
| <Box flex={{direction: 'column'}}> | ||
| <Tooltip | ||
| key={column.name} | ||
| position="bottom-left" | ||
| content={ | ||
| <div style={{maxWidth: 500}}> | ||
| <Description | ||
| maxHeight={400} | ||
| description={column.description || 'No description provided'} | ||
| /> | ||
| </div> | ||
| } | ||
| > | ||
| <ColumnLink | ||
| to={assetDetailsPathForKey(assetKey, {view: 'lineage', column: column.name})} | ||
| $blueBackground={blueBackground} | ||
| > | ||
| {icon ? <Icon name={icon} /> : <span style={{width: 16}} />} | ||
| <Caption style={{whiteSpace: 'nowrap', overflow: 'hidden', textOverflow: 'ellipsis'}}> | ||
| {column.name} | ||
| </Caption> | ||
| </ColumnLink> | ||
| </Tooltip> | ||
| </Box> | ||
| ); | ||
| }; | ||
|
|
||
| const ColumnLink = styled(Link)<{$blueBackground: boolean}>` | ||
| height: 28px; | ||
| margin: 2px 12px; | ||
| padding-left: 2px; | ||
| padding-right: 4px; | ||
| display: flex; | ||
| gap: 4px; | ||
| align-items: center; | ||
| transition: background 100ms linear; | ||
| border-radius: 8px; | ||
| ${StyledTag} { | ||
| background: none; | ||
| color: ${Colors.textLight()}; | ||
| } | ||
| ${(p) => | ||
| p.$blueBackground | ||
| ? ` | ||
| background: ${Colors.backgroundBlue()}` | ||
| : ` | ||
| &:hover { | ||
| text-decoration: none; | ||
| background: ${Colors.backgroundLightHover()}; | ||
| }`} | ||
| `; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.