diff --git a/js_modules/dagster-ui/packages/ui-components/src/components/Icon.tsx b/js_modules/dagster-ui/packages/ui-components/src/components/Icon.tsx index 0ae0152f826a2..405c33c0ffc4f 100644 --- a/js_modules/dagster-ui/packages/ui-components/src/components/Icon.tsx +++ b/js_modules/dagster-ui/packages/ui-components/src/components/Icon.tsx @@ -42,6 +42,7 @@ import chevron_right from '../icon-svgs/chevron_right.svg'; import close from '../icon-svgs/close.svg'; import code_location from '../icon-svgs/code_location.svg'; import collapse_arrows from '../icon-svgs/collapse_arrows.svg'; +import column_lineage from '../icon-svgs/column_lineage.svg'; import concept_book from '../icon-svgs/concept-book.svg'; import console_icon from '../icon-svgs/console.svg'; import content_copy from '../icon-svgs/content_copy.svg'; @@ -284,6 +285,7 @@ export const Icons = { console: console_icon, content_copy, collapse_arrows, + column_lineage, corporate_fare, delete: deleteSVG, done, diff --git a/js_modules/dagster-ui/packages/ui-components/src/icon-svgs/column_lineage.svg b/js_modules/dagster-ui/packages/ui-components/src/icon-svgs/column_lineage.svg new file mode 100644 index 0000000000000..c1e6ff95e4075 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-components/src/icon-svgs/column_lineage.svg @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/js_modules/dagster-ui/packages/ui-core/package.json b/js_modules/dagster-ui/packages/ui-core/package.json index ac3b9e7f64a5d..0cf029f8e0df5 100644 --- a/js_modules/dagster-ui/packages/ui-core/package.json +++ b/js_modules/dagster-ui/packages/ui-core/package.json @@ -43,7 +43,7 @@ "codemirror": "^5.65.2", "color": "^3.0.0", "cronstrue": "^1.84.0", - "dagre": "^0.8.5", + "dagre": "dagster-io/dagre#0.8.5", "date-fns": "^2.28.0", "dayjs": "^1.11.7", "deepmerge": "^4.2.2", diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetColumnsNode.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetColumnsNode.tsx new file mode 100644 index 0000000000000..af83205e3fb16 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetColumnsNode.tsx @@ -0,0 +1,115 @@ +import {Box, Caption, Colors, Icon, StyledTag, Tooltip} from '@dagster-io/ui-components'; +import {Link} from 'react-router-dom'; +import styled from 'styled-components'; + +import { + AssetInsetForHoverEffect, + AssetNameRow, + AssetNodeBox, + AssetNodeContainer, +} from './AssetNode'; +import {AssetNodeFragment} from './types/AssetNode.types'; +import {Timestamp} from '../app/time/Timestamp'; +import {assetDetailsPathForKey} from '../assets/assetDetailsPathForKey'; +import {AssetColumnLineageLocalColumn} from '../assets/lineage/useColumnLineageDataForAssets'; +import {AssetComputeKindTag} from '../graph/OpTags'; +import {AssetKeyInput} from '../graphql/types'; +import {iconForColumnType} from '../metadata/TableSchema'; +import {Description} from '../pipelines/Description'; + +export const AssetColumnsGroupNode = ({ + selected, + definition, + height, + asOf, +}: { + selected: boolean; + definition: AssetNodeFragment; + asOf: string | undefined; + height: number; +}) => { + return ( + + +
+ + + + + {asOf ? ( + + + + ) : undefined} + + + + + + ); +}; + +export const AssetColumnNode = ({ + assetKey, + column, + blueBackground, +}: { + assetKey: AssetKeyInput; + column: AssetColumnLineageLocalColumn; + blueBackground: boolean; +}) => { + const icon = iconForColumnType(column.type ?? ''); + + return ( + + + +
+ } + > + + {icon ? : } + + {column.name} + + + + + ); +}; + +const ColumnLink = styled(Link)<{$blueBackground: boolean}>` + height: 28px; + margin: 2px 12px; + padding-left: 2px; + padding-right: 4px; + display: flex; + gap: 4px; + align-items: center; + transition: background 100ms linear; + border-radius: 8px; + + ${StyledTag} { + background: none; + color: ${Colors.textLight()}; + } + ${(p) => + p.$blueBackground + ? ` + background: ${Colors.backgroundBlue()}` + : ` + &:hover { + text-decoration: none; + background: ${Colors.backgroundLightHover()}; + }`} +`; diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx index e52795efc9103..9e1e6ff3d86de 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx @@ -16,6 +16,7 @@ import pickBy from 'lodash/pickBy'; import uniq from 'lodash/uniq'; import without from 'lodash/without'; import * as React from 'react'; +import {useMemo} from 'react'; import styled from 'styled-components'; import {AssetEdges} from './AssetEdges'; @@ -269,7 +270,11 @@ const AssetGraphExplorerWithData = ({ }); const focusGroupIdAfterLayoutRef = React.useRef(''); - const {layout, loading, async} = useAssetLayout(assetGraphData, expandedGroups, direction); + const {layout, loading, async} = useAssetLayout( + assetGraphData, + expandedGroups, + useMemo(() => ({direction}), [direction]), + ); React.useEffect(() => { if (!loading) { diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetNode.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetNode.tsx index 2386d26b5448e..72f356b602588 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetNode.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetNode.tsx @@ -27,13 +27,15 @@ interface Props { } export const AssetNode = React.memo(({definition, selected}: Props) => { - const displayName = definition.assetKey.path[definition.assetKey.path.length - 1]!; const isSource = definition.isSource; const {liveData} = useAssetLiveData(definition.assetKey); return ( - + { - - - - -
- {withMiddleTruncation(displayName, { - maxLength: ASSET_NODE_NAME_MAX_LENGTH, - })} -
-
- + {definition.description ? ( @@ -81,6 +69,28 @@ export const AssetNode = React.memo(({definition, selected}: Props) => { ); }, isEqual); +export const AssetNameRow = ({definition}: {definition: AssetNodeFragment}) => { + const displayName = definition.assetKey.path[definition.assetKey.path.length - 1]!; + + return ( + + + + +
+ {withMiddleTruncation(displayName, { + maxLength: ASSET_NODE_NAME_MAX_LENGTH, + })} +
+
+ + ); +}; + const AssetNodeRowBox = styled(Box)` white-space: nowrap; line-height: 12px; @@ -244,7 +254,7 @@ export const ASSET_NODE_FRAGMENT = gql` } `; -const AssetInsetForHoverEffect = styled.div` +export const AssetInsetForHoverEffect = styled.div` padding: 10px 4px 2px 4px; height: 100%; @@ -253,7 +263,7 @@ const AssetInsetForHoverEffect = styled.div` } `; -const AssetNodeContainer = styled.div<{$selected: boolean}>` +export const AssetNodeContainer = styled.div<{$selected: boolean}>` user-select: none; cursor: pointer; padding: 6px; @@ -264,7 +274,11 @@ const AssetNodeShowOnHover = styled.span` display: none; `; -const AssetNodeBox = styled.div<{$isSource: boolean; $selected: boolean}>` +export const AssetNodeBox = styled.div<{ + $isSource: boolean; + $selected: boolean; + $noScale?: boolean; +}>` ${(p) => p.$isSource ? `border: 2px dashed ${p.$selected ? Colors.accentGrayHover() : Colors.accentGray()}` @@ -280,7 +294,7 @@ const AssetNodeBox = styled.div<{$isSource: boolean; $selected: boolean}>` &:hover { ${(p) => !p.$selected && `border: 2px solid ${Colors.lineageNodeBorderHover()};`}; box-shadow: ${Colors.shadowDefault()} 0px 1px 4px 0px; - scale: 1.03; + scale: ${(p) => (p.$noScale ? '1' : '1.03')}; ${AssetNodeShowOnHover} { display: initial; } diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/SidebarAssetInfo.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/SidebarAssetInfo.tsx index 3ce4087a9cbb3..c6644d6256854 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/SidebarAssetInfo.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/SidebarAssetInfo.tsx @@ -32,6 +32,7 @@ import { import {DagsterTypeSummary} from '../dagstertype/DagsterType'; import {DagsterTypeFragment} from '../dagstertype/types/DagsterType.types'; import {METADATA_ENTRY_FRAGMENT} from '../metadata/MetadataEntry'; +import {TableSchemaLineageContext} from '../metadata/TableSchema'; import {Description} from '../pipelines/Description'; import {SidebarSection, SidebarTitle} from '../pipelines/SidebarComponents'; import {ResourceContainer, ResourceHeader} from '../pipelines/SidebarOpHelpers'; @@ -159,7 +160,12 @@ export const SidebarAssetInfo = ({graphNode}: {graphNode: GraphNode}) => { {assetMetadata.length > 0 && ( - + + + )} diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/Utils.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/Utils.tsx index 03b3191d33a90..f8489806cd64c 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/Utils.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/Utils.tsx @@ -44,7 +44,7 @@ export function isHiddenAssetGroupJob(jobName: string) { // export type GraphId = string; export const toGraphId = (key: {path: string[]}): GraphId => JSON.stringify(key.path); -export const fromGraphID = (graphId: GraphId): AssetNodeKeyFragment => ({ +export const fromGraphId = (graphId: GraphId): AssetNodeKeyFragment => ({ path: JSON.parse(graphId), __typename: 'AssetKey', }); @@ -266,7 +266,7 @@ export const itemWithAssetKey = (key: {path: string[]}) => { return (asset: {assetKey: {path: string[]}}) => tokenForAssetKey(asset.assetKey) === token; }; -export const isGroupId = (str: string) => /^[^@:]+@[^@:]+:[^@:]+$/.test(str); +export const isGroupId = (str: string) => /^[^@:]+@[^@:]+:.+$/.test(str); export const groupIdForNode = (node: GraphNode) => [ @@ -281,7 +281,7 @@ export const groupIdForNode = (node: GraphNode) => export const getUpstreamNodes = memoize( (assetKey: AssetNodeKeyFragment, graphData: GraphData): AssetNodeKeyFragment[] => { const upstream = Object.keys(graphData.upstream[toGraphId(assetKey)] || {}); - const currentUpstream = upstream.map((graphId) => fromGraphID(graphId)); + const currentUpstream = upstream.map((graphId) => fromGraphId(graphId)); return [ assetKey, ...currentUpstream, diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/layout.ts b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/layout.ts index c3b7ba28b882d..c4083745196fb 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/layout.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/layout.ts @@ -35,8 +35,59 @@ export type AssetGraphLayout = { }; const MARGIN = 100; +export type LayoutAssetGraphConfig = dagre.GraphLabel & { + direction: AssetLayoutDirection; + /** Pass `auto` to use getAssetNodeDimensions, or a value to give nodes a fixed height */ + nodeHeight: number | 'auto'; + /** Our asset groups have "title bars" - use these numbers to adjust the bounding boxes. + * Note that these adjustments are applied post-dagre layout. For padding > nodesep, you + * may need to set "clusterpaddingtop", "clusterpaddingbottom" so Dagre lays out the boxes + * with more spacing. + */ + groupPaddingTop: number; + groupPaddingBottom: number; + groupRendering: 'if-varied' | 'always'; + + /** Supported in Dagre, just not documented. Additional spacing between group nodes */ + clusterpaddingtop?: number; + clusterpaddingbottom?: number; +}; + export type LayoutAssetGraphOptions = { direction: AssetLayoutDirection; + overrides?: Partial; +}; + +export const Config = { + horizontal: { + ranker: 'tight-tree', + direction: 'horizontal', + marginx: MARGIN, + marginy: MARGIN, + ranksep: 60, + rankdir: 'LR', + edgesep: 90, + nodesep: -10, + nodeHeight: 'auto', + groupPaddingTop: 65, + groupPaddingBottom: -15, + groupRendering: 'if-varied', + clusterpaddingtop: 100, + }, + vertical: { + ranker: 'tight-tree', + direction: 'horizontal', + marginx: MARGIN, + marginy: MARGIN, + ranksep: 20, + rankdir: 'TB', + nodesep: 40, + edgesep: 10, + nodeHeight: 'auto', + groupPaddingTop: 40, + groupPaddingBottom: -20, + groupRendering: 'if-varied', + }, }; export const layoutAssetGraph = ( @@ -44,30 +95,9 @@ export const layoutAssetGraph = ( opts: LayoutAssetGraphOptions, ): AssetGraphLayout => { const g = new dagre.graphlib.Graph({compound: true}); + const config = Object.assign({}, Config[opts.direction], opts.overrides || {}); - const ranker = 'tight-tree'; - - g.setGraph( - opts.direction === 'horizontal' - ? { - rankdir: 'LR', - marginx: MARGIN, - marginy: MARGIN, - nodesep: -10, - edgesep: 90, - ranksep: 60, - ranker, - } - : { - rankdir: 'TB', - marginx: MARGIN, - marginy: MARGIN, - nodesep: 40, - edgesep: 10, - ranksep: 20, - ranker, - }, - ); + g.setGraph(config); g.setDefaultEdgeLabel(() => ({})); // const shouldRender = (node?: GraphNode) => node && node.definition.opNames.length > 0; @@ -92,13 +122,17 @@ export const layoutAssetGraph = ( } // Add all the group boxes to the graph - const groupsPresent = Object.keys(groups).length > 1; + const groupsPresent = + config.groupRendering === 'if-varied' ? Object.keys(groups).length > 1 : true; + if (groupsPresent) { Object.keys(groups).forEach((groupId) => { if (expandedGroups.includes(groupId)) { - g.setNode(groupId, {}); // sized based on it's children + // sized based on it's children, but "border" tells Dagre we want cluster-level + // spacing between the node and others. Necessary because our groups have title bars. + g.setNode(groupId, {borderType: 'borderRight'}); } else { - g.setNode(groupId, {width: 320, height: 110}); + g.setNode(groupId, {width: ASSET_NODE_WIDTH, height: 110}); } }); } @@ -106,7 +140,12 @@ export const layoutAssetGraph = ( // Add all the nodes inside expanded groups to the graph renderedNodes.forEach((node) => { if (!groupsPresent || expandedGroups.includes(groupIdForNode(node))) { - g.setNode(node.id, getAssetNodeDimensions(node.definition)); + const label = + config.nodeHeight === 'auto' + ? getAssetNodeDimensions(node.definition) + : {width: ASSET_NODE_WIDTH, height: config.nodeHeight}; + + g.setNode(node.id, label); if (groupsPresent && node.definition.groupName) { g.setParent(node.id, groupIdForNode(node)); } @@ -205,10 +244,11 @@ export const layoutAssetGraph = ( } for (const group of Object.values(groups)) { if (group.expanded) { - group.bounds = - opts.direction === 'horizontal' - ? padBounds(group.bounds, {x: 15, top: 65, bottom: -15}) - : padBounds(group.bounds, {x: 15, top: 40, bottom: -20}); + group.bounds = padBounds(group.bounds, { + x: 15, + top: config.groupPaddingTop, + bottom: config.groupPaddingBottom, + }); } } } @@ -276,6 +316,7 @@ export const extendBounds = (a: IBounds, b: IBounds) => { return {x: xmin, y: ymin, width: xmax - xmin, height: ymax - ymin}; }; +export const ASSET_NODE_WIDTH = 320; export const ASSET_NODE_NAME_MAX_LENGTH = 38; export const getAssetNodeDimensions = (def: { @@ -289,7 +330,7 @@ export const getAssetNodeDimensions = (def: { computeKind: string | null; changedReasons?: ChangeReason[]; }) => { - const width = 320; + const width = ASSET_NODE_WIDTH; let height = 100; // top tags area + name + description diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetColumnLineageGraph.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetColumnLineageGraph.tsx new file mode 100644 index 0000000000000..37a6c90fe9371 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetColumnLineageGraph.tsx @@ -0,0 +1,166 @@ +import {Box, Spinner} from '@dagster-io/ui-components'; +import {useMemo, useRef, useState} from 'react'; +import styled from 'styled-components'; + +import {SVGSaveZoomLevel, useLastSavedZoomLevel} from './SavedZoomLevel'; +import {AssetColumnLineages} from './lineage/useColumnLineageDataForAssets'; +import {fromColumnGraphId, useColumnLineageLayout} from './useColumnLineageLayout'; +import {AssetColumnNode, AssetColumnsGroupNode} from '../asset-graph/AssetColumnsNode'; +import {AssetEdges} from '../asset-graph/AssetEdges'; +import {AssetNodeContextMenuWrapper} from '../asset-graph/AssetNode'; +import {GraphData, fromGraphId, toGraphId} from '../asset-graph/Utils'; +import {DEFAULT_MAX_ZOOM, SVGViewport} from '../graph/SVGViewport'; +import {isNodeOffscreen} from '../graph/common'; +import {AssetKeyInput} from '../graphql/types'; + +export const AssetColumnLineageGraph = ({ + assetKey, + assetGraphData, + columnLineageData, + focusedColumn, +}: { + assetKey: AssetKeyInput; + assetGraphData: GraphData; + columnLineageData: AssetColumnLineages; + focusedColumn: string; +}) => { + const focusedAssetGraphId = toGraphId(assetKey); + + const [highlighted, setHighlighted] = useState(null); + + const {layout, loading} = useColumnLineageLayout( + assetGraphData, + focusedAssetGraphId, + focusedColumn, + columnLineageData, + ); + + const viewportEl = useRef(); + + useLastSavedZoomLevel(viewportEl, layout, focusedAssetGraphId); + + const blue = useMemo(() => { + const blue = new Set(); + if (!highlighted || !layout) { + return blue; + } + + for (const id of highlighted) { + blue.add(id); + layout.edges.filter((e) => e.fromId === id).forEach((e) => blue.add(e.toId)); + layout.edges.filter((e) => e.toId === id).forEach((e) => blue.add(e.fromId)); + } + return blue; + }, [layout, highlighted]); + + if (!layout || loading) { + return ( + + + + ); + } + + return ( + (viewportEl.current = r || undefined)} + interactor={SVGViewport.Interactors.PanAndZoom} + defaultZoom="zoom-to-fit" + graphWidth={layout.width} + graphHeight={layout.height} + onDoubleClick={(e) => { + viewportEl.current?.autocenter(true); + e.stopPropagation(); + }} + maxZoom={DEFAULT_MAX_ZOOM} + maxAutocenterZoom={DEFAULT_MAX_ZOOM} + > + {({scale}, viewportRect) => ( + + {viewportEl.current && } + + {Object.values(layout.groups) + .filter((node) => !isNodeOffscreen(node.bounds, viewportRect)) + .map(({id, bounds}) => { + const groupAssetGraphId = toGraphId({path: id.split(':').pop()!.split('>')}); + const graphNode = assetGraphData.nodes[groupAssetGraphId]; + const contextMenuProps = { + graphData: assetGraphData, + node: graphNode!, + }; + + const cols = columnLineageData[groupAssetGraphId] || {}; + const colsAsOf = Object.values(cols)[0]?.asOf; + + return ( + setHighlighted([id])} + onMouseLeave={() => setHighlighted(null)} + onDoubleClick={(e) => { + viewportEl.current?.zoomToSVGBox(bounds, true, 1.2); + e.stopPropagation(); + }} + > + + + + + ); + })} + + + + {Object.values(layout.nodes) + .filter((node) => !isNodeOffscreen(node.bounds, viewportRect)) + .map(({id, bounds}) => { + const {assetGraphId, column} = fromColumnGraphId(id); + const assetKey = fromGraphId(assetGraphId); + + const col = columnLineageData[assetGraphId]?.[column] || { + name: column, + description: 'Not found in column metadata', + type: null, + upstream: [], + asOf: undefined, + }; + + return ( + setHighlighted([id])} + onMouseLeave={() => setHighlighted(null)} + onDoubleClick={(e) => { + viewportEl.current?.zoomToSVGBox(bounds, true, 1.2); + e.stopPropagation(); + }} + > + + + ); + })} + + )} + + ); +}; + +const SVGContainer = styled.svg` + overflow: visible; + border-radius: 0; +`; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetEventMetadataEntriesTable.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetEventMetadataEntriesTable.tsx index 1aa46aafaf8df..32ffaa4168861 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetEventMetadataEntriesTable.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetEventMetadataEntriesTable.tsx @@ -11,7 +11,7 @@ import { } from './types/useRecentAssetEvents.types'; import {Timestamp} from '../app/time/Timestamp'; import {HIDDEN_METADATA_ENTRY_LABELS, MetadataEntry} from '../metadata/MetadataEntry'; -import {isCanonicalTableSchemaEntry} from '../metadata/TableSchema'; +import {isCanonicalColumnLineageEntry, isCanonicalColumnSchemaEntry} from '../metadata/TableSchema'; import {MetadataEntryFragment} from '../metadata/types/MetadataEntry.types'; import {titleForRun} from '../runs/RunUtils'; @@ -89,14 +89,14 @@ export const AssetEventMetadataEntriesTable = ({ const filteredRows = useMemo( () => - allRows.filter( - (row) => - !filter || - row.entry.label.toLowerCase().includes(filter.toLowerCase()) || - !HIDDEN_METADATA_ENTRY_LABELS.has(row.entry.label) || - !hideTableSchema || - !isCanonicalTableSchemaEntry(row.entry), - ), + allRows + .filter((row) => !filter || row.entry.label.toLowerCase().includes(filter.toLowerCase())) + .filter( + (row) => + !HIDDEN_METADATA_ENTRY_LABELS.has(row.entry.label) && + !(isCanonicalColumnSchemaEntry(row.entry) && hideTableSchema) && + !isCanonicalColumnLineageEntry(row.entry), + ), [allRows, filter, hideTableSchema], ); diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineage.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineage.tsx index 69fff663b3b46..04306e03958b8 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineage.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineage.tsx @@ -5,17 +5,24 @@ import { Colors, Icon, JoinedButtons, + MenuItem, + Suggest, TextInput, } from '@dagster-io/ui-components'; -import {useEffect, useMemo, useState} from 'react'; +import React, {useEffect, useMemo, useState} from 'react'; import styled from 'styled-components'; +import {AssetColumnLineageGraph} from './AssetColumnLineageGraph'; import {AssetNodeLineageGraph} from './AssetNodeLineageGraph'; import {LaunchAssetExecutionButton} from './LaunchAssetExecutionButton'; +import {asAssetKeyInput} from './asInput'; +import {useColumnLineageDataForAssets} from './lineage/useColumnLineageDataForAssets'; import {AssetLineageScope, AssetViewParams} from './types'; -import {GraphData} from '../asset-graph/Utils'; +import {GraphData, toGraphId} from '../asset-graph/Utils'; import {AssetGraphQueryItem, calculateGraphDistances} from '../asset-graph/useAssetGraphData'; import {AssetKeyInput} from '../graphql/types'; +import {useQueryPersistedState} from '../hooks/useQueryPersistedState'; +import {ClearButton} from '../ui/ClearButton'; export const AssetNodeLineage = ({ params, @@ -45,6 +52,18 @@ export const AssetNodeLineage = ({ const currentDepth = Math.max(1, Math.min(maxDepth, requestedDepth)); + const assetGraphKeys = useMemo( + () => Object.values(assetGraphData.nodes).map(asAssetKeyInput), + [assetGraphData], + ); + const columnLineageData = useColumnLineageDataForAssets(assetGraphKeys); + const columnLineage = columnLineageData[toGraphId(assetKey)]; + const [column, setColumn] = useQueryPersistedState({ + queryKey: 'column', + decode: (qs) => qs.column || null, + encode: (column) => ({column: column || undefined}), + }); + return ( setParams({...params, lineageDepth: depth})} max={maxDepth} /> + {columnLineage || column ? ( + <> + Column + setColumn(null)} + style={{marginTop: 5, marginRight: 4}} + > + + + ) : undefined, + }} + selectedItem={column} + items={Object.keys(columnLineage || {})} + noResults="No matching columns" + onItemSelect={setColumn} + inputValueRenderer={(item) => item} + itemPredicate={(query, item) => + item.toLocaleLowerCase().includes(query.toLocaleLowerCase()) + } + itemRenderer={(item, itemProps) => ( + itemProps.handleClick(e)} + text={item} + key={item} + /> + )} + /> + + ) : undefined}
{Object.values(assetGraphData.nodes).length > 1 ? ( )} - + {column ? ( + + ) : ( + + )} ); }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineageGraph.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineageGraph.tsx index 13d99ed05388b..7d9df75ba23ab 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineageGraph.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineageGraph.tsx @@ -1,8 +1,9 @@ import {Box, Spinner} from '@dagster-io/ui-components'; -import {useEffect, useMemo, useRef, useState} from 'react'; +import {useMemo, useRef, useState} from 'react'; import {useHistory} from 'react-router-dom'; import styled from 'styled-components'; +import {SVGSaveZoomLevel, useLastSavedZoomLevel} from './SavedZoomLevel'; import {assetDetailsPathForKey} from './assetDetailsPathForKey'; import {AssetKey, AssetViewParams} from './types'; import {AssetEdges} from '../asset-graph/AssetEdges'; @@ -11,13 +12,13 @@ import {AssetNode, AssetNodeContextMenuWrapper, AssetNodeMinimal} from '../asset import {ExpandedGroupNode, GroupOutline} from '../asset-graph/ExpandedGroupNode'; import {AssetNodeLink} from '../asset-graph/ForeignNode'; import {GraphData, GraphNode, groupIdForNode, toGraphId} from '../asset-graph/Utils'; +import {LayoutAssetGraphOptions} from '../asset-graph/layout'; import {DEFAULT_MAX_ZOOM, SVGViewport} from '../graph/SVGViewport'; import {useAssetLayout} from '../graph/asyncGraphLayout'; import {isNodeOffscreen} from '../graph/common'; import {AssetKeyInput} from '../graphql/types'; -import {getJSONForKey} from '../hooks/useStateWithStorage'; -const LINEAGE_GRAPH_ZOOM_LEVEL = 'lineageGraphZoomLevel'; +const LINEAGE_GRAPH_OPTIONS: LayoutAssetGraphOptions = {direction: 'horizontal'}; export const AssetNodeLineageGraph = ({ assetKey, @@ -42,7 +43,7 @@ export const AssetNodeLineageGraph = ({ const [highlighted, setHighlighted] = useState(null); - const {layout, loading} = useAssetLayout(assetGraphData, allGroups, 'horizontal'); + const {layout, loading} = useAssetLayout(assetGraphData, allGroups, LINEAGE_GRAPH_OPTIONS); const viewportEl = useRef(); const history = useHistory(); @@ -50,13 +51,7 @@ export const AssetNodeLineageGraph = ({ history.push(assetDetailsPathForKey(key, {...params, lineageScope: 'neighbors'})); }; - useEffect(() => { - if (viewportEl.current && layout) { - const lastZoomLevel = Number(getJSONForKey(LINEAGE_GRAPH_ZOOM_LEVEL)); - viewportEl.current.autocenter(false, lastZoomLevel); - viewportEl.current.focus(); - } - }, [viewportEl, layout, assetGraphId]); + useLastSavedZoomLevel(viewportEl, layout, assetGraphId); if (!layout || loading) { return ( @@ -113,10 +108,7 @@ export const AssetNodeLineageGraph = ({ .map((group) => ( @@ -174,17 +166,6 @@ export const AssetNodeLineageGraph = ({ ); }; -const SVGSaveZoomLevel = ({scale}: {scale: number}) => { - useEffect(() => { - try { - window.localStorage.setItem(LINEAGE_GRAPH_ZOOM_LEVEL, JSON.stringify(scale)); - } catch (err) { - // no-op - } - }, [scale]); - return <>; -}; - const SVGContainer = styled.svg` overflow: visible; border-radius: 0; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeOverview.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeOverview.tsx index 3d99f1881739b..7fb5e537cf297 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeOverview.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeOverview.tsx @@ -36,6 +36,7 @@ import {SimpleStakeholderAssetStatus} from './SimpleStakeholderAssetStatus'; import {UnderlyingOpsOrGraph} from './UnderlyingOpsOrGraph'; import {AssetChecksStatusSummary} from './asset-checks/AssetChecksStatusSummary'; import {assetDetailsPathForKey} from './assetDetailsPathForKey'; +import {buildConsolidatedColumnSchema} from './buildConsolidatedColumnSchema'; import {globalAssetGraphPathForAssetsAndDescendants} from './globalAssetGraphPathToString'; import {AssetKey} from './types'; import {AssetNodeDefinitionFragment} from './types/AssetNodeDefinition.types'; @@ -55,7 +56,11 @@ import {DagsterTypeSummary} from '../dagstertype/DagsterType'; import {AssetComputeKindTag} from '../graph/OpTags'; import {useStateWithStorage} from '../hooks/useStateWithStorage'; import {useLaunchPadHooks} from '../launchpad/LaunchpadHooksContext'; -import {TableSchema, isCanonicalTableSchemaEntry} from '../metadata/TableSchema'; +import { + TableSchema, + TableSchemaLineageContext, + isCanonicalColumnLineageEntry, +} from '../metadata/TableSchema'; import {RepositoryLink} from '../nav/RepositoryLink'; import {ScheduleOrSensorTag} from '../nav/ScheduleOrSensorTag'; import {useRepositoryLocationForAddress} from '../nav/useRepositoryLocationForAddress'; @@ -102,35 +107,13 @@ export const AssetNodeOverview = ({ return ; } - const materializationTableSchema = materialization?.metadataEntries.find( - isCanonicalTableSchemaEntry, - ); - const materializationTableSchemaLoadTimestamp = materialization - ? Number(materialization.timestamp) - : undefined; - const definitionTableSchema = assetNode?.metadataEntries.find(isCanonicalTableSchemaEntry); - const definitionTableSchemaLoadTimestamp = assetNodeLoadTimestamp; - - let tableSchema = materializationTableSchema ?? definitionTableSchema; - const tableSchemaLoadTimestamp = - materializationTableSchemaLoadTimestamp ?? definitionTableSchemaLoadTimestamp; - - // Merge the descriptions from the definition table schema with the materialization table schema - if (materializationTableSchema && definitionTableSchema) { - const definitionTableSchemaColumnDescriptionsByName = Object.fromEntries( - definitionTableSchema.schema.columns.map((column) => [column.name, column.description]), - ); - const mergedColumns = materializationTableSchema.schema.columns.map((column) => { - const description = - definitionTableSchemaColumnDescriptionsByName[column.name] || column.description; - return {...column, description}; - }); - - tableSchema = { - ...materializationTableSchema, - schema: {...materializationTableSchema.schema, columns: mergedColumns}, - }; - } + const {tableSchema, tableSchemaLoadTimestamp} = buildConsolidatedColumnSchema({ + materialization, + definition: assetNode, + definitionLoadTimestamp: assetNodeLoadTimestamp, + }); + + const columnSchema = materialization?.metadataEntries.find(isCanonicalColumnLineageEntry); const renderStatusSection = () => ( @@ -411,10 +394,14 @@ export const AssetNodeOverview = ({ {tableSchema && ( - + + + )} diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx index 219673952c2e2..01033017d8822 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx @@ -366,7 +366,7 @@ function getQueryForVisibleAssets( return {query: `+"${token}"+`, requestedDepth: 1}; } if (view === 'lineage') { - const defaultDepth = lineageScope === 'neighbors' ? 2 : 5; + const defaultDepth = 1; const requestedDepth = Number(lineageDepth) || defaultDepth; const depthStr = '+'.repeat(requestedDepth); diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/LastMaterializationMetadata.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/LastMaterializationMetadata.tsx index 6c3c9559b59c6..63263400d7b5a 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/LastMaterializationMetadata.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/LastMaterializationMetadata.tsx @@ -15,6 +15,7 @@ import {Timestamp} from '../app/time/Timestamp'; import {LiveDataForNode, isHiddenAssetGroupJob} from '../asset-graph/Utils'; import {AssetKeyInput} from '../graphql/types'; import {MetadataEntry} from '../metadata/MetadataEntry'; +import {isCanonicalColumnLineageEntry} from '../metadata/TableSchema'; import {Description} from '../pipelines/Description'; import {PipelineReference} from '../pipelines/PipelineReference'; import {linkToRunEvent, titleForRun} from '../runs/RunUtils'; @@ -156,18 +157,20 @@ export const LatestMaterializationMetadata = ({ ) : null} - {latestEvent.metadataEntries.map((entry) => ( - - {entry.label} - - - - - ))} + {latestEvent.metadataEntries + .filter((entry) => !isCanonicalColumnLineageEntry(entry)) + .map((entry) => ( + + {entry.label} + + + + + ))} ) : ( diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/SavedZoomLevel.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/SavedZoomLevel.tsx new file mode 100644 index 0000000000000..3d75adb38f5c0 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/SavedZoomLevel.tsx @@ -0,0 +1,31 @@ +import {MutableRefObject, useEffect} from 'react'; + +import {SVGViewport} from '../graph/SVGViewport'; +import {getJSONForKey} from '../hooks/useStateWithStorage'; + +const LINEAGE_GRAPH_ZOOM_LEVEL = 'lineageGraphZoomLevel'; + +export const SVGSaveZoomLevel = ({scale}: {scale: number}) => { + useEffect(() => { + try { + window.localStorage.setItem(LINEAGE_GRAPH_ZOOM_LEVEL, JSON.stringify(scale)); + } catch (err) { + // no-op + } + }, [scale]); + return <>; +}; + +export function useLastSavedZoomLevel( + viewportEl: MutableRefObject, + layout: import('../asset-graph/layout').AssetGraphLayout | null, + graphFocusChangeKey: string, +) { + useEffect(() => { + if (viewportEl.current && layout) { + const lastZoomLevel = Number(getJSONForKey(LINEAGE_GRAPH_ZOOM_LEVEL)); + viewportEl.current.autocenter(false, lastZoomLevel); + viewportEl.current.focus(); + } + }, [viewportEl, layout, graphFocusChangeKey]); +} diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/buildConsolidatedColumnSchema.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/buildConsolidatedColumnSchema.tsx new file mode 100644 index 0000000000000..c4081b0699bb0 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/buildConsolidatedColumnSchema.tsx @@ -0,0 +1,51 @@ +// eslint-disable-next-line no-restricted-imports + +import {AssetColumnLineageQuery} from './lineage/types/useColumnLineageDataForAssets.types'; +import {isCanonicalColumnSchemaEntry} from '../metadata/TableSchema'; + +type AssetDefinitionWithMetadata = AssetColumnLineageQuery['assetNodes'][0]; + +/** + * This helper pulls the `columns` metadata entry from the most recent materialization + * and the asset definition, blending the two together to produce the most current + * representation. (Sometimes descriptions are only in the definition-time version) + */ +export function buildConsolidatedColumnSchema({ + materialization, + definition, + definitionLoadTimestamp, +}: { + materialization: + | Pick + | undefined; + definition: Pick | undefined; + definitionLoadTimestamp: number | undefined; +}) { + const materializationTableSchema = materialization?.metadataEntries.find( + isCanonicalColumnSchemaEntry, + ); + const materializationTimestamp = materialization ? Number(materialization.timestamp) : undefined; + const definitionTableSchema = definition?.metadataEntries.find(isCanonicalColumnSchemaEntry); + + let tableSchema = materializationTableSchema ?? definitionTableSchema; + const tableSchemaLoadTimestamp = materializationTimestamp ?? definitionLoadTimestamp; + + // Merge the descriptions from the definition table schema with the materialization table schema + if (materializationTableSchema && definitionTableSchema) { + const definitionTableSchemaColumnDescriptionsByName = Object.fromEntries( + definitionTableSchema.schema.columns.map((column) => [column.name, column.description]), + ); + const mergedColumns = materializationTableSchema.schema.columns.map((column) => { + const description = + definitionTableSchemaColumnDescriptionsByName[column.name] || column.description; + return {...column, description}; + }); + + tableSchema = { + ...materializationTableSchema, + schema: {...materializationTableSchema.schema, columns: mergedColumns}, + }; + } + console.log(tableSchema); + return {tableSchema, tableSchemaLoadTimestamp}; +} diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/lineage/types/useColumnLineageDataForAssets.types.ts b/js_modules/dagster-ui/packages/ui-core/src/assets/lineage/types/useColumnLineageDataForAssets.types.ts new file mode 100644 index 0000000000000..e9db8c746bf02 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/lineage/types/useColumnLineageDataForAssets.types.ts @@ -0,0 +1,82 @@ +// Generated GraphQL types, do not edit manually. + +import * as Types from '../../../graphql/types'; + +export type AssetColumnLineageQueryVariables = Types.Exact<{ + assetKeys: Array | Types.AssetKeyInput; +}>; + +export type AssetColumnLineageQuery = { + __typename: 'Query'; + assetNodes: Array<{ + __typename: 'AssetNode'; + id: string; + assetKey: {__typename: 'AssetKey'; path: Array}; + metadataEntries: Array< + | {__typename: 'AssetMetadataEntry'; label: string} + | {__typename: 'BoolMetadataEntry'; label: string} + | {__typename: 'FloatMetadataEntry'; label: string} + | {__typename: 'IntMetadataEntry'; label: string} + | {__typename: 'JobMetadataEntry'; label: string} + | {__typename: 'JsonMetadataEntry'; label: string} + | {__typename: 'MarkdownMetadataEntry'; label: string} + | {__typename: 'NotebookMetadataEntry'; label: string} + | {__typename: 'NullMetadataEntry'; label: string} + | {__typename: 'PathMetadataEntry'; label: string} + | {__typename: 'PipelineRunMetadataEntry'; label: string} + | {__typename: 'PythonArtifactMetadataEntry'; label: string} + | {__typename: 'TableMetadataEntry'; label: string} + | { + __typename: 'TableSchemaMetadataEntry'; + label: string; + schema: { + __typename: 'TableSchema'; + columns: Array<{ + __typename: 'TableColumn'; + name: string; + type: string; + description: string | null; + }>; + }; + } + | {__typename: 'TextMetadataEntry'; label: string} + | {__typename: 'TimestampMetadataEntry'; label: string} + | {__typename: 'UrlMetadataEntry'; label: string} + >; + assetMaterializations: Array<{ + __typename: 'MaterializationEvent'; + timestamp: string; + metadataEntries: Array< + | {__typename: 'AssetMetadataEntry'; label: string} + | {__typename: 'BoolMetadataEntry'; label: string} + | {__typename: 'FloatMetadataEntry'; label: string} + | {__typename: 'IntMetadataEntry'; label: string} + | {__typename: 'JobMetadataEntry'; label: string} + | {__typename: 'JsonMetadataEntry'; jsonString: string; label: string} + | {__typename: 'MarkdownMetadataEntry'; label: string} + | {__typename: 'NotebookMetadataEntry'; label: string} + | {__typename: 'NullMetadataEntry'; label: string} + | {__typename: 'PathMetadataEntry'; label: string} + | {__typename: 'PipelineRunMetadataEntry'; label: string} + | {__typename: 'PythonArtifactMetadataEntry'; label: string} + | {__typename: 'TableMetadataEntry'; label: string} + | { + __typename: 'TableSchemaMetadataEntry'; + label: string; + schema: { + __typename: 'TableSchema'; + columns: Array<{ + __typename: 'TableColumn'; + name: string; + type: string; + description: string | null; + }>; + }; + } + | {__typename: 'TextMetadataEntry'; label: string} + | {__typename: 'TimestampMetadataEntry'; label: string} + | {__typename: 'UrlMetadataEntry'; label: string} + >; + }>; + }>; +}; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/lineage/useColumnLineageDataForAssets.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/lineage/useColumnLineageDataForAssets.tsx new file mode 100644 index 0000000000000..5140b7a95af8d --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/lineage/useColumnLineageDataForAssets.tsx @@ -0,0 +1,158 @@ +import {gql, useApolloClient} from '@apollo/client'; +import React, {useMemo, useRef, useState} from 'react'; + +import { + AssetColumnLineageQuery, + AssetColumnLineageQueryVariables, +} from './types/useColumnLineageDataForAssets.types'; +import {toGraphId} from '../../asset-graph/Utils'; +import {AssetKeyInput} from '../../graphql/types'; +import {isCanonicalColumnLineageEntry} from '../../metadata/TableSchema'; +import {buildConsolidatedColumnSchema} from '../buildConsolidatedColumnSchema'; + +export type AssetColumnLineageServer = { + [column: string]: { + // Note: This is [["key_part_1", "key_part_2"]] but the outer array + // only contains one item, it's a serialization odditiy. + upstream_asset_key: string[][]; + upstream_column_name: string; + }[]; +}; + +export type AssetColumnLineageLocalColumn = { + name: string; + type: string | null; + description: string | null; + asOf: string | undefined; // materialization timestamp + upstream: { + assetKey: AssetKeyInput; + columnName: string; + }[]; +}; + +export type AssetColumnLineageLocal = { + [column: string]: AssetColumnLineageLocalColumn; +}; + +export type AssetColumnLineages = {[graphId: string]: AssetColumnLineageLocal | undefined}; + +/** + * The column definitions and the column lineage are in two separate metadata entries, + * and the definitions may be specified in definition-time or materialization-time metadata. + * Parse them both and combine the results into a single representation of asset columns + * that is easier for the rest of the front-end to use. + */ +const getColumnLineage = ( + asset: AssetColumnLineageQuery['assetNodes'][0], +): AssetColumnLineageLocal | undefined => { + const materialization = asset.assetMaterializations[0]; + const lineageMetadata = materialization?.metadataEntries.find(isCanonicalColumnLineageEntry); + if (!lineageMetadata) { + return undefined; + } + + const {tableSchema} = buildConsolidatedColumnSchema({ + materialization, + definition: asset, + definitionLoadTimestamp: undefined, + }); + + const lineageParsed: AssetColumnLineageServer = JSON.parse(lineageMetadata.jsonString); + const schemaParsed = tableSchema?.schema + ? Object.fromEntries(tableSchema.schema.columns.map((col) => [col.name, col])) + : {}; + + return Object.fromEntries( + Object.entries(lineageParsed).map(([column, m]) => [ + column, + { + name: column, + asOf: materialization?.timestamp, + type: schemaParsed[column]?.type || null, + description: schemaParsed[column]?.description || null, + upstream: m.map((u) => ({ + assetKey: {path: u.upstream_asset_key[0]!}, + columnName: u.upstream_column_name, + })), + }, + ]), + ); +}; + +export function useColumnLineageDataForAssets(assetKeys: AssetKeyInput[]) { + const [loaded, setLoaded] = useState({}); + const client = useApolloClient(); + const fetching = useRef(false); + const missing = useMemo( + () => assetKeys.filter((a) => !loaded[toGraphId(a)]), + [assetKeys, loaded], + ); + + React.useEffect(() => { + const fetch = async () => { + fetching.current = true; + const {data} = await client.query({ + query: ASSET_COLUMN_LINEAGE_QUERY, + variables: {assetKeys: missing}, + }); + fetching.current = false; + + setLoaded((loaded) => ({ + ...loaded, + ...Object.fromEntries( + data.assetNodes.map((n) => [toGraphId(n.assetKey), getColumnLineage(n)]), + ), + })); + }; + if (!fetching.current && missing.length) { + void fetch(); + } + }, [client, missing]); + + return loaded; +} + +const ASSET_COLUMN_LINEAGE_QUERY = gql` + query AssetColumnLineage($assetKeys: [AssetKeyInput!]!) { + assetNodes(loadMaterializations: true, assetKeys: $assetKeys) { + id + assetKey { + path + } + metadataEntries { + __typename + label + ... on TableSchemaMetadataEntry { + label + schema { + columns { + name + type + description + } + } + } + } + assetMaterializations(limit: 1) { + timestamp + metadataEntries { + __typename + label + ... on TableSchemaMetadataEntry { + label + schema { + columns { + name + type + description + } + } + } + ... on JsonMetadataEntry { + jsonString + } + } + } + } + } +`; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/types.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/types.tsx index b61cf71a59ed6..b87c0bfaabdf1 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/types.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/types.tsx @@ -21,4 +21,5 @@ export interface AssetViewParams { evaluation?: string; checkDetail?: string; default_range?: string; + column?: string; } diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/useColumnLineageLayout.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/useColumnLineageLayout.tsx new file mode 100644 index 0000000000000..376b60e6a069a --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/useColumnLineageLayout.tsx @@ -0,0 +1,142 @@ +import {useMemo} from 'react'; + +import {AssetColumnLineages} from './lineage/useColumnLineageDataForAssets'; +import {GraphData, groupIdForNode, toGraphId, tokenForAssetKey} from '../asset-graph/Utils'; +import {LayoutAssetGraphOptions} from '../asset-graph/layout'; +import {useAssetLayout} from '../graph/asyncGraphLayout'; + +const LINEAGE_GRAPH_COLUMN_LAYOUT_OPTIONS: LayoutAssetGraphOptions = { + direction: 'horizontal', + overrides: { + nodeHeight: 32, + nodesep: 0, + edgesep: 0, + clusterpaddingtop: 50, + groupPaddingBottom: -3, + groupPaddingTop: 68, + groupRendering: 'always', + }, +}; + +type Item = {assetGraphId: string; column: string; direction?: 'upstream' | 'downstream'}; + +export function toColumnGraphId(item: {assetGraphId: string; column: string}) { + return JSON.stringify({assetGraphId: item.assetGraphId, column: item.column}); +} +export function fromColumnGraphId(id: string) { + return JSON.parse(id) as {assetGraphId: string; column: string}; +} + +/** + * This function returns GraphData in which each `node` is a column of an asset and each `group` + * is an asset, essentially "zooming in" to the asset column level. This is a bit awkward but + * allows us to reuse the asset layout engine (and all it's caching, async dispatch, etc) for + * this view. + */ +export function useColumnLineageLayout( + assetGraphData: GraphData, + assetGraphId: string, + column: string, + columnLineageData: AssetColumnLineages, +) { + const {columnGraphData, groups} = useMemo(() => { + const columnGraphData: GraphData = { + nodes: {}, + downstream: {}, + upstream: {}, + }; + + const downstreams = buildReverseEdgeLookupTable(columnLineageData); + + const addEdge = (upstreamId: string, downstreamId: string) => { + columnGraphData.upstream[downstreamId] = columnGraphData.upstream[downstreamId] || {}; + columnGraphData.upstream[downstreamId]![upstreamId] = true; + columnGraphData.downstream[upstreamId] = columnGraphData.downstream[upstreamId] || {}; + columnGraphData.downstream[upstreamId]![downstreamId] = true; + }; + + const groups = new Set(); + const queue: Item[] = [{assetGraphId, column}]; + let item: Item | undefined; + + while ((item = queue.pop())) { + if (!item) { + continue; + } + const id = toColumnGraphId(item); + const {assetGraphId, column, direction} = item; + const assetNode = assetGraphData.nodes[assetGraphId]; + if (columnGraphData.nodes[id] || !assetNode) { + continue; // visited already + } + + const columnGraphNode = { + id, + assetKey: assetNode.assetKey, + definition: { + ...assetNode.definition, + groupName: `${tokenForAssetKey(assetNode.assetKey)}`, + }, + }; + columnGraphData.nodes[id] = columnGraphNode; + groups.add(groupIdForNode(columnGraphNode)); + + if (!direction || direction === 'upstream') { + const lineageForColumn = columnLineageData[assetGraphId]?.[column]; + for (const upstream of lineageForColumn?.upstream || []) { + const upstreamGraphId = toGraphId(upstream.assetKey); + const upstreamItem: Item = { + assetGraphId: upstreamGraphId, + column: upstream.columnName, + direction: 'upstream', + }; + if (assetGraphData.nodes[upstreamItem.assetGraphId]) { + queue.push(upstreamItem); + addEdge(toColumnGraphId(upstreamItem), id); + } + } + } + if (!direction || direction === 'downstream') { + for (const downstreamId of Object.keys(downstreams[id] || {})) { + const downstreamItem: Item = { + ...fromColumnGraphId(downstreamId), + direction: 'downstream', + }; + if (assetGraphData.nodes[downstreamItem.assetGraphId]) { + queue.push(downstreamItem); + addEdge(id, downstreamId); + } + } + } + } + + return {columnGraphData, groups: Array.from(groups)}; + }, [assetGraphData, column, columnLineageData, assetGraphId]); + + return useAssetLayout(columnGraphData, groups, LINEAGE_GRAPH_COLUMN_LAYOUT_OPTIONS); +} + +/** + * The column lineage data we get from asset metadata only gives us upstreams for each column. + * To efficiently build graph data we need both upstreams and downstreams for each column. + * This function visits every node and builds a downstreams lookup table. + */ +function buildReverseEdgeLookupTable(columnLineageData: AssetColumnLineages) { + const downstreams: {[id: string]: {[id: string]: true}} = {}; + + Object.entries(columnLineageData).forEach(([downstreamAssetGraphId, e]) => { + Object.entries(e || {}).forEach(([downstreamColumnName, {upstream}]) => { + const downstreamKey = toColumnGraphId({ + assetGraphId: downstreamAssetGraphId, + column: downstreamColumnName, + }); + for (const {assetKey, columnName} of upstream) { + const key = toColumnGraphId({assetGraphId: toGraphId(assetKey), column: columnName}); + downstreams[key] = downstreams[key] || {}; + downstreams[key]![downstreamKey] = true; + } + }); + }); + + return downstreams; +} diff --git a/js_modules/dagster-ui/packages/ui-core/src/graph/asyncGraphLayout.ts b/js_modules/dagster-ui/packages/ui-core/src/graph/asyncGraphLayout.ts index d7ec4bf980477..fa47bbe03099b 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/graph/asyncGraphLayout.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/graph/asyncGraphLayout.ts @@ -5,12 +5,7 @@ import {ILayoutOp, LayoutOpGraphOptions, OpGraphLayout, layoutOpGraph} from './l import {useFeatureFlags} from '../app/Flags'; import {asyncMemoize, indexedDBAsyncMemoize} from '../app/Util'; import {GraphData} from '../asset-graph/Utils'; -import { - AssetGraphLayout, - AssetLayoutDirection, - LayoutAssetGraphOptions, - layoutAssetGraph, -} from '../asset-graph/layout'; +import {AssetGraphLayout, LayoutAssetGraphOptions, layoutAssetGraph} from '../asset-graph/layout'; const ASYNC_LAYOUT_SOLID_COUNT = 50; @@ -186,14 +181,13 @@ export function useOpLayout(ops: ILayoutOp[], parentOp?: ILayoutOp) { export function useAssetLayout( _graphData: GraphData, expandedGroups: string[], - direction: AssetLayoutDirection, + opts: LayoutAssetGraphOptions, ) { const [state, dispatch] = useReducer(reducer, initialState); const flags = useFeatureFlags(); const graphData = useMemo(() => ({..._graphData, expandedGroups}), [expandedGroups, _graphData]); - const opts = useMemo(() => ({direction}), [direction]); const cacheKey = _assetLayoutCacheKey(graphData, opts); const nodeCount = Object.keys(graphData.nodes).length; const runAsync = nodeCount >= ASYNC_LAYOUT_SOLID_COUNT; diff --git a/js_modules/dagster-ui/packages/ui-core/src/metadata/MetadataEntry.tsx b/js_modules/dagster-ui/packages/ui-core/src/metadata/MetadataEntry.tsx index 4f44ea1b94c79..a50b1a299139a 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/metadata/MetadataEntry.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/metadata/MetadataEntry.tsx @@ -223,6 +223,7 @@ export const MetadataEntry = ({ ) : ( JSON.stringify(entry.schema, null, 2)} content={() => ( React.ReactNode; copyContent: () => string; }) => { @@ -377,7 +379,7 @@ const MetadataEntryModalAction = (props: { setOpen(true)}>{props.children} setOpen(false)} isOpen={open} diff --git a/js_modules/dagster-ui/packages/ui-core/src/metadata/TableSchema.tsx b/js_modules/dagster-ui/packages/ui-core/src/metadata/TableSchema.tsx index 7982db4f315e0..839e83cd86460 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/metadata/TableSchema.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/metadata/TableSchema.tsx @@ -11,12 +11,20 @@ import { Tooltip, } from '@dagster-io/ui-components'; import {Spacing} from '@dagster-io/ui-components/src/components/types'; -import {useState} from 'react'; +import {createContext, useContext, useState} from 'react'; import {TableSchemaFragment} from './types/TableSchema.types'; import {Timestamp} from '../app/time/Timestamp'; import {StyledTableWithHeader} from '../assets/AssetEventMetadataEntriesTable'; -import {MaterializationEvent, TableSchemaMetadataEntry} from '../graphql/types'; +import {assetDetailsPathForKey} from '../assets/assetDetailsPathForKey'; +import { + AssetKeyInput, + JsonMetadataEntry, + MaterializationEvent, + TableSchemaMetadataEntry, +} from '../graphql/types'; +import {Description} from '../pipelines/Description'; +import {AnchorButton} from '../ui/AnchorButton'; type ITableSchema = TableSchemaFragment; @@ -28,16 +36,26 @@ interface ITableSchemaProps { itemHorizontalPadding?: Spacing; } -export const isCanonicalTableSchemaEntry = ( +export const isCanonicalColumnSchemaEntry = ( m: Pick, ): m is TableSchemaMetadataEntry => m.__typename === 'TableSchemaMetadataEntry' && m.label === 'dagster/column_schema'; +export const isCanonicalColumnLineageEntry = ( + m: Pick, +): m is JsonMetadataEntry => m.__typename === 'JsonMetadataEntry' && m.label === 'lineage'; + +export const TableSchemaLineageContext = createContext<{assetKey: AssetKeyInput | null}>({ + assetKey: null, +}); + export const TableSchema = ({ schema, schemaLoadTimestamp, itemHorizontalPadding, }: ITableSchemaProps) => { + const {assetKey} = useContext(TableSchemaLineageContext); + const multiColumnConstraints = schema.constraints?.other || []; const [filter, setFilter] = useState(''); const rows = schema.columns.filter( @@ -76,6 +94,7 @@ export const TableSchema = ({ Column name Type Description + {assetKey ? : undefined} @@ -85,14 +104,29 @@ export const TableSchema = ({ {column.name} - + {!column.constraints.nullable && NonNullableTag} {column.constraints.unique && UniqueTag} {column.constraints.other.map((constraint, i) => ( ))} - {column.description} + + + + {assetKey ? ( + + + } + to={assetDetailsPathForKey(assetKey, { + view: 'lineage', + column: column.name, + })} + /> + + + ) : undefined} ))} {rows.length === 0 && ( @@ -108,7 +142,7 @@ export const TableSchema = ({ ); }; -const iconForType = (type: string): IconName | null => { +export const iconForColumnType = (type: string): IconName | null => { const lower = type.toLowerCase(); if (lower.includes('bool')) { return 'datatype_bool'; @@ -128,12 +162,14 @@ const iconForType = (type: string): IconName | null => { return null; }; -const TypeTag = ({type = '', icon}: {type: string; icon: IconName | null}) => { +export const TypeTag = ({type = ''}: {type: string}) => { if (type.trim().replace(/\?/g, '').length === 0) { // Do not render type '' or '?' or any other empty value. return ; } + const icon = iconForColumnType(type); + return ( diff --git a/js_modules/dagster-ui/patches/dagre+0.8.5.patch b/js_modules/dagster-ui/patches/dagre+0.8.5.patch deleted file mode 100644 index fab38165260b3..0000000000000 --- a/js_modules/dagster-ui/patches/dagre+0.8.5.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff --git a/node_modules/dagre/lib/order/index.js b/node_modules/dagre/lib/order/index.js -index 4ac2d9f..a2182fe 100644 ---- a/node_modules/dagre/lib/order/index.js -+++ b/node_modules/dagre/lib/order/index.js -@@ -73,7 +73,9 @@ function sweepLayerGraphs(layerGraphs, biasRight) { - function assignOrder(g, layering) { - _.forEach(layering, function(layer) { - _.forEach(layer, function(v, i) { -- g.node(v).order = i; -+ try { -+ g.node(v).order = i; -+ } catch (e) {} - }); - }); - } diff --git a/js_modules/dagster-ui/yarn.lock b/js_modules/dagster-ui/yarn.lock index f44fbb20646dc..fb58b04e57528 100644 --- a/js_modules/dagster-ui/yarn.lock +++ b/js_modules/dagster-ui/yarn.lock @@ -2598,7 +2598,7 @@ __metadata: codemirror: "npm:^5.65.2" color: "npm:^3.0.0" cronstrue: "npm:^1.84.0" - dagre: "npm:^0.8.5" + dagre: "dagster-io/dagre#0.8.5" date-fns: "npm:^2.28.0" dayjs: "npm:^1.11.7" deepmerge: "npm:^4.2.2" @@ -11528,13 +11528,13 @@ __metadata: languageName: node linkType: hard -"dagre@npm:^0.8.5": +"dagre@dagster-io/dagre#0.8.5": version: 0.8.5 - resolution: "dagre@npm:0.8.5" + resolution: "dagre@https://github.com/dagster-io/dagre.git#commit=c2a1821cc7f8a220e819461b82b6ddbf48189100" dependencies: graphlib: "npm:^2.1.8" lodash: "npm:^4.17.15" - checksum: f39899e29e9090581d67177ef6e2dd3ca5d7f764fbf3de81758d879bba66fee6fd8802d41d0c5d3d9a0563b334e99e1454a8d6ab4ce17e8e4f50836a3a403fdd + checksum: 6a94d8d9b1c3132b406b5921fd2bbd1a207c78fc1048216787fec68fad1d96f649cb084d1ae576f5456532a4275f22a113d140265f05937db60e5918a25adac5 languageName: node linkType: hard