diff --git a/js_modules/dagster-ui/packages/ui-components/src/components/Icon.tsx b/js_modules/dagster-ui/packages/ui-components/src/components/Icon.tsx index 405c33c0ffc4f..0ae0152f826a2 100644 --- a/js_modules/dagster-ui/packages/ui-components/src/components/Icon.tsx +++ b/js_modules/dagster-ui/packages/ui-components/src/components/Icon.tsx @@ -42,7 +42,6 @@ import chevron_right from '../icon-svgs/chevron_right.svg'; import close from '../icon-svgs/close.svg'; import code_location from '../icon-svgs/code_location.svg'; import collapse_arrows from '../icon-svgs/collapse_arrows.svg'; -import column_lineage from '../icon-svgs/column_lineage.svg'; import concept_book from '../icon-svgs/concept-book.svg'; import console_icon from '../icon-svgs/console.svg'; import content_copy from '../icon-svgs/content_copy.svg'; @@ -285,7 +284,6 @@ export const Icons = { console: console_icon, content_copy, collapse_arrows, - column_lineage, corporate_fare, delete: deleteSVG, done, diff --git a/js_modules/dagster-ui/packages/ui-components/src/icon-svgs/column_lineage.svg b/js_modules/dagster-ui/packages/ui-components/src/icon-svgs/column_lineage.svg deleted file mode 100644 index c1e6ff95e4075..0000000000000 --- a/js_modules/dagster-ui/packages/ui-components/src/icon-svgs/column_lineage.svg +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/js_modules/dagster-ui/packages/ui-core/package.json b/js_modules/dagster-ui/packages/ui-core/package.json index 0cf029f8e0df5..ac3b9e7f64a5d 100644 --- a/js_modules/dagster-ui/packages/ui-core/package.json +++ b/js_modules/dagster-ui/packages/ui-core/package.json @@ -43,7 +43,7 @@ "codemirror": "^5.65.2", "color": "^3.0.0", "cronstrue": "^1.84.0", - "dagre": "dagster-io/dagre#0.8.5", + "dagre": "^0.8.5", "date-fns": "^2.28.0", "dayjs": "^1.11.7", "deepmerge": "^4.2.2", diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetColumnsNode.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetColumnsNode.tsx deleted file mode 100644 index af83205e3fb16..0000000000000 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetColumnsNode.tsx +++ /dev/null @@ -1,115 +0,0 @@ -import {Box, Caption, Colors, Icon, StyledTag, Tooltip} from '@dagster-io/ui-components'; -import {Link} from 'react-router-dom'; -import styled from 'styled-components'; - -import { - AssetInsetForHoverEffect, - AssetNameRow, - AssetNodeBox, - AssetNodeContainer, -} from './AssetNode'; -import {AssetNodeFragment} from './types/AssetNode.types'; -import {Timestamp} from '../app/time/Timestamp'; -import {assetDetailsPathForKey} from '../assets/assetDetailsPathForKey'; -import {AssetColumnLineageLocalColumn} from '../assets/lineage/useColumnLineageDataForAssets'; -import {AssetComputeKindTag} from '../graph/OpTags'; -import {AssetKeyInput} from '../graphql/types'; -import {iconForColumnType} from '../metadata/TableSchema'; -import {Description} from '../pipelines/Description'; - -export const AssetColumnsGroupNode = ({ - selected, - definition, - height, - asOf, -}: { - selected: boolean; - definition: AssetNodeFragment; - asOf: string | undefined; - height: number; -}) => { - return ( - - -
- - - - - {asOf ? ( - - - - ) : undefined} - - - - - - ); -}; - -export const AssetColumnNode = ({ - assetKey, - column, - blueBackground, -}: { - assetKey: AssetKeyInput; - column: AssetColumnLineageLocalColumn; - blueBackground: boolean; -}) => { - const icon = iconForColumnType(column.type ?? ''); - - return ( - - - -
- } - > - - {icon ? : } - - {column.name} - - - - - ); -}; - -const ColumnLink = styled(Link)<{$blueBackground: boolean}>` - height: 28px; - margin: 2px 12px; - padding-left: 2px; - padding-right: 4px; - display: flex; - gap: 4px; - align-items: center; - transition: background 100ms linear; - border-radius: 8px; - - ${StyledTag} { - background: none; - color: ${Colors.textLight()}; - } - ${(p) => - p.$blueBackground - ? ` - background: ${Colors.backgroundBlue()}` - : ` - &:hover { - text-decoration: none; - background: ${Colors.backgroundLightHover()}; - }`} -`; diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx index 9e1e6ff3d86de..e52795efc9103 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx @@ -16,7 +16,6 @@ import pickBy from 'lodash/pickBy'; import uniq from 'lodash/uniq'; import without from 'lodash/without'; import * as React from 'react'; -import {useMemo} from 'react'; import styled from 'styled-components'; import {AssetEdges} from './AssetEdges'; @@ -270,11 +269,7 @@ const AssetGraphExplorerWithData = ({ }); const focusGroupIdAfterLayoutRef = React.useRef(''); - const {layout, loading, async} = useAssetLayout( - assetGraphData, - expandedGroups, - useMemo(() => ({direction}), [direction]), - ); + const {layout, loading, async} = useAssetLayout(assetGraphData, expandedGroups, direction); React.useEffect(() => { if (!loading) { diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetNode.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetNode.tsx index 70641ee82ef90..2148119e451cf 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetNode.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetNode.tsx @@ -28,16 +28,14 @@ interface Props { } export const AssetNode = React.memo(({definition, selected}: Props) => { + const displayName = definition.assetKey.path[definition.assetKey.path.length - 1]!; const isSource = definition.isSource; const {liveData} = useAssetLiveData(definition.assetKey); return ( - + { - + + + + +
+ {withMiddleTruncation(displayName, { + maxLength: ASSET_NODE_NAME_MAX_LENGTH, + })} +
+
+ {definition.description ? ( @@ -71,28 +83,6 @@ export const AssetNode = React.memo(({definition, selected}: Props) => { ); }, isEqual); -export const AssetNameRow = ({definition}: {definition: AssetNodeFragment}) => { - const displayName = definition.assetKey.path[definition.assetKey.path.length - 1]!; - - return ( - - - - -
- {withMiddleTruncation(displayName, { - maxLength: ASSET_NODE_NAME_MAX_LENGTH, - })} -
-
- - ); -}; - const AssetNodeRowBox = styled(Box)` white-space: nowrap; line-height: 12px; @@ -254,7 +244,7 @@ export const ASSET_NODE_FRAGMENT = gql` } `; -export const AssetInsetForHoverEffect = styled.div` +const AssetInsetForHoverEffect = styled.div` padding: 10px 4px 2px 4px; height: 100%; @@ -263,7 +253,7 @@ export const AssetInsetForHoverEffect = styled.div` } `; -export const AssetNodeContainer = styled.div<{$selected: boolean}>` +const AssetNodeContainer = styled.div<{$selected: boolean}>` user-select: none; cursor: pointer; padding: 6px; @@ -274,11 +264,7 @@ const AssetNodeShowOnHover = styled.span` display: none; `; -export const AssetNodeBox = styled.div<{ - $isSource: boolean; - $selected: boolean; - $noScale?: boolean; -}>` +const AssetNodeBox = styled.div<{$isSource: boolean; $selected: boolean}>` ${(p) => p.$isSource ? `border: 2px dashed ${p.$selected ? Colors.accentGrayHover() : Colors.accentGray()}` @@ -294,7 +280,7 @@ export const AssetNodeBox = styled.div<{ &:hover { ${(p) => !p.$selected && `border: 2px solid ${Colors.lineageNodeBorderHover()};`}; box-shadow: ${Colors.shadowDefault()} 0px 1px 4px 0px; - scale: ${(p) => (p.$noScale ? '1' : '1.03')}; + scale: 1.03; ${AssetNodeShowOnHover} { display: initial; } diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/SidebarAssetInfo.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/SidebarAssetInfo.tsx index c6644d6256854..3ce4087a9cbb3 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/SidebarAssetInfo.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/SidebarAssetInfo.tsx @@ -32,7 +32,6 @@ import { import {DagsterTypeSummary} from '../dagstertype/DagsterType'; import {DagsterTypeFragment} from '../dagstertype/types/DagsterType.types'; import {METADATA_ENTRY_FRAGMENT} from '../metadata/MetadataEntry'; -import {TableSchemaLineageContext} from '../metadata/TableSchema'; import {Description} from '../pipelines/Description'; import {SidebarSection, SidebarTitle} from '../pipelines/SidebarComponents'; import {ResourceContainer, ResourceHeader} from '../pipelines/SidebarOpHelpers'; @@ -160,12 +159,7 @@ export const SidebarAssetInfo = ({graphNode}: {graphNode: GraphNode}) => { {assetMetadata.length > 0 && ( - - - + )} diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/Utils.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/Utils.tsx index f8489806cd64c..03b3191d33a90 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/Utils.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/Utils.tsx @@ -44,7 +44,7 @@ export function isHiddenAssetGroupJob(jobName: string) { // export type GraphId = string; export const toGraphId = (key: {path: string[]}): GraphId => JSON.stringify(key.path); -export const fromGraphId = (graphId: GraphId): AssetNodeKeyFragment => ({ +export const fromGraphID = (graphId: GraphId): AssetNodeKeyFragment => ({ path: JSON.parse(graphId), __typename: 'AssetKey', }); @@ -266,7 +266,7 @@ export const itemWithAssetKey = (key: {path: string[]}) => { return (asset: {assetKey: {path: string[]}}) => tokenForAssetKey(asset.assetKey) === token; }; -export const isGroupId = (str: string) => /^[^@:]+@[^@:]+:.+$/.test(str); +export const isGroupId = (str: string) => /^[^@:]+@[^@:]+:[^@:]+$/.test(str); export const groupIdForNode = (node: GraphNode) => [ @@ -281,7 +281,7 @@ export const groupIdForNode = (node: GraphNode) => export const getUpstreamNodes = memoize( (assetKey: AssetNodeKeyFragment, graphData: GraphData): AssetNodeKeyFragment[] => { const upstream = Object.keys(graphData.upstream[toGraphId(assetKey)] || {}); - const currentUpstream = upstream.map((graphId) => fromGraphId(graphId)); + const currentUpstream = upstream.map((graphId) => fromGraphID(graphId)); return [ assetKey, ...currentUpstream, diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/layout.ts b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/layout.ts index c4083745196fb..c3b7ba28b882d 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/layout.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/layout.ts @@ -35,59 +35,8 @@ export type AssetGraphLayout = { }; const MARGIN = 100; -export type LayoutAssetGraphConfig = dagre.GraphLabel & { - direction: AssetLayoutDirection; - /** Pass `auto` to use getAssetNodeDimensions, or a value to give nodes a fixed height */ - nodeHeight: number | 'auto'; - /** Our asset groups have "title bars" - use these numbers to adjust the bounding boxes. - * Note that these adjustments are applied post-dagre layout. For padding > nodesep, you - * may need to set "clusterpaddingtop", "clusterpaddingbottom" so Dagre lays out the boxes - * with more spacing. - */ - groupPaddingTop: number; - groupPaddingBottom: number; - groupRendering: 'if-varied' | 'always'; - - /** Supported in Dagre, just not documented. Additional spacing between group nodes */ - clusterpaddingtop?: number; - clusterpaddingbottom?: number; -}; - export type LayoutAssetGraphOptions = { direction: AssetLayoutDirection; - overrides?: Partial; -}; - -export const Config = { - horizontal: { - ranker: 'tight-tree', - direction: 'horizontal', - marginx: MARGIN, - marginy: MARGIN, - ranksep: 60, - rankdir: 'LR', - edgesep: 90, - nodesep: -10, - nodeHeight: 'auto', - groupPaddingTop: 65, - groupPaddingBottom: -15, - groupRendering: 'if-varied', - clusterpaddingtop: 100, - }, - vertical: { - ranker: 'tight-tree', - direction: 'horizontal', - marginx: MARGIN, - marginy: MARGIN, - ranksep: 20, - rankdir: 'TB', - nodesep: 40, - edgesep: 10, - nodeHeight: 'auto', - groupPaddingTop: 40, - groupPaddingBottom: -20, - groupRendering: 'if-varied', - }, }; export const layoutAssetGraph = ( @@ -95,9 +44,30 @@ export const layoutAssetGraph = ( opts: LayoutAssetGraphOptions, ): AssetGraphLayout => { const g = new dagre.graphlib.Graph({compound: true}); - const config = Object.assign({}, Config[opts.direction], opts.overrides || {}); - g.setGraph(config); + const ranker = 'tight-tree'; + + g.setGraph( + opts.direction === 'horizontal' + ? { + rankdir: 'LR', + marginx: MARGIN, + marginy: MARGIN, + nodesep: -10, + edgesep: 90, + ranksep: 60, + ranker, + } + : { + rankdir: 'TB', + marginx: MARGIN, + marginy: MARGIN, + nodesep: 40, + edgesep: 10, + ranksep: 20, + ranker, + }, + ); g.setDefaultEdgeLabel(() => ({})); // const shouldRender = (node?: GraphNode) => node && node.definition.opNames.length > 0; @@ -122,17 +92,13 @@ export const layoutAssetGraph = ( } // Add all the group boxes to the graph - const groupsPresent = - config.groupRendering === 'if-varied' ? Object.keys(groups).length > 1 : true; - + const groupsPresent = Object.keys(groups).length > 1; if (groupsPresent) { Object.keys(groups).forEach((groupId) => { if (expandedGroups.includes(groupId)) { - // sized based on it's children, but "border" tells Dagre we want cluster-level - // spacing between the node and others. Necessary because our groups have title bars. - g.setNode(groupId, {borderType: 'borderRight'}); + g.setNode(groupId, {}); // sized based on it's children } else { - g.setNode(groupId, {width: ASSET_NODE_WIDTH, height: 110}); + g.setNode(groupId, {width: 320, height: 110}); } }); } @@ -140,12 +106,7 @@ export const layoutAssetGraph = ( // Add all the nodes inside expanded groups to the graph renderedNodes.forEach((node) => { if (!groupsPresent || expandedGroups.includes(groupIdForNode(node))) { - const label = - config.nodeHeight === 'auto' - ? getAssetNodeDimensions(node.definition) - : {width: ASSET_NODE_WIDTH, height: config.nodeHeight}; - - g.setNode(node.id, label); + g.setNode(node.id, getAssetNodeDimensions(node.definition)); if (groupsPresent && node.definition.groupName) { g.setParent(node.id, groupIdForNode(node)); } @@ -244,11 +205,10 @@ export const layoutAssetGraph = ( } for (const group of Object.values(groups)) { if (group.expanded) { - group.bounds = padBounds(group.bounds, { - x: 15, - top: config.groupPaddingTop, - bottom: config.groupPaddingBottom, - }); + group.bounds = + opts.direction === 'horizontal' + ? padBounds(group.bounds, {x: 15, top: 65, bottom: -15}) + : padBounds(group.bounds, {x: 15, top: 40, bottom: -20}); } } } @@ -316,7 +276,6 @@ export const extendBounds = (a: IBounds, b: IBounds) => { return {x: xmin, y: ymin, width: xmax - xmin, height: ymax - ymin}; }; -export const ASSET_NODE_WIDTH = 320; export const ASSET_NODE_NAME_MAX_LENGTH = 38; export const getAssetNodeDimensions = (def: { @@ -330,7 +289,7 @@ export const getAssetNodeDimensions = (def: { computeKind: string | null; changedReasons?: ChangeReason[]; }) => { - const width = ASSET_NODE_WIDTH; + const width = 320; let height = 100; // top tags area + name + description diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetColumnLineageGraph.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetColumnLineageGraph.tsx deleted file mode 100644 index 37a6c90fe9371..0000000000000 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetColumnLineageGraph.tsx +++ /dev/null @@ -1,166 +0,0 @@ -import {Box, Spinner} from '@dagster-io/ui-components'; -import {useMemo, useRef, useState} from 'react'; -import styled from 'styled-components'; - -import {SVGSaveZoomLevel, useLastSavedZoomLevel} from './SavedZoomLevel'; -import {AssetColumnLineages} from './lineage/useColumnLineageDataForAssets'; -import {fromColumnGraphId, useColumnLineageLayout} from './useColumnLineageLayout'; -import {AssetColumnNode, AssetColumnsGroupNode} from '../asset-graph/AssetColumnsNode'; -import {AssetEdges} from '../asset-graph/AssetEdges'; -import {AssetNodeContextMenuWrapper} from '../asset-graph/AssetNode'; -import {GraphData, fromGraphId, toGraphId} from '../asset-graph/Utils'; -import {DEFAULT_MAX_ZOOM, SVGViewport} from '../graph/SVGViewport'; -import {isNodeOffscreen} from '../graph/common'; -import {AssetKeyInput} from '../graphql/types'; - -export const AssetColumnLineageGraph = ({ - assetKey, - assetGraphData, - columnLineageData, - focusedColumn, -}: { - assetKey: AssetKeyInput; - assetGraphData: GraphData; - columnLineageData: AssetColumnLineages; - focusedColumn: string; -}) => { - const focusedAssetGraphId = toGraphId(assetKey); - - const [highlighted, setHighlighted] = useState(null); - - const {layout, loading} = useColumnLineageLayout( - assetGraphData, - focusedAssetGraphId, - focusedColumn, - columnLineageData, - ); - - const viewportEl = useRef(); - - useLastSavedZoomLevel(viewportEl, layout, focusedAssetGraphId); - - const blue = useMemo(() => { - const blue = new Set(); - if (!highlighted || !layout) { - return blue; - } - - for (const id of highlighted) { - blue.add(id); - layout.edges.filter((e) => e.fromId === id).forEach((e) => blue.add(e.toId)); - layout.edges.filter((e) => e.toId === id).forEach((e) => blue.add(e.fromId)); - } - return blue; - }, [layout, highlighted]); - - if (!layout || loading) { - return ( - - - - ); - } - - return ( - (viewportEl.current = r || undefined)} - interactor={SVGViewport.Interactors.PanAndZoom} - defaultZoom="zoom-to-fit" - graphWidth={layout.width} - graphHeight={layout.height} - onDoubleClick={(e) => { - viewportEl.current?.autocenter(true); - e.stopPropagation(); - }} - maxZoom={DEFAULT_MAX_ZOOM} - maxAutocenterZoom={DEFAULT_MAX_ZOOM} - > - {({scale}, viewportRect) => ( - - {viewportEl.current && } - - {Object.values(layout.groups) - .filter((node) => !isNodeOffscreen(node.bounds, viewportRect)) - .map(({id, bounds}) => { - const groupAssetGraphId = toGraphId({path: id.split(':').pop()!.split('>')}); - const graphNode = assetGraphData.nodes[groupAssetGraphId]; - const contextMenuProps = { - graphData: assetGraphData, - node: graphNode!, - }; - - const cols = columnLineageData[groupAssetGraphId] || {}; - const colsAsOf = Object.values(cols)[0]?.asOf; - - return ( - setHighlighted([id])} - onMouseLeave={() => setHighlighted(null)} - onDoubleClick={(e) => { - viewportEl.current?.zoomToSVGBox(bounds, true, 1.2); - e.stopPropagation(); - }} - > - - - - - ); - })} - - - - {Object.values(layout.nodes) - .filter((node) => !isNodeOffscreen(node.bounds, viewportRect)) - .map(({id, bounds}) => { - const {assetGraphId, column} = fromColumnGraphId(id); - const assetKey = fromGraphId(assetGraphId); - - const col = columnLineageData[assetGraphId]?.[column] || { - name: column, - description: 'Not found in column metadata', - type: null, - upstream: [], - asOf: undefined, - }; - - return ( - setHighlighted([id])} - onMouseLeave={() => setHighlighted(null)} - onDoubleClick={(e) => { - viewportEl.current?.zoomToSVGBox(bounds, true, 1.2); - e.stopPropagation(); - }} - > - - - ); - })} - - )} - - ); -}; - -const SVGContainer = styled.svg` - overflow: visible; - border-radius: 0; -`; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetEventMetadataEntriesTable.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetEventMetadataEntriesTable.tsx index 32ffaa4168861..1aa46aafaf8df 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetEventMetadataEntriesTable.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetEventMetadataEntriesTable.tsx @@ -11,7 +11,7 @@ import { } from './types/useRecentAssetEvents.types'; import {Timestamp} from '../app/time/Timestamp'; import {HIDDEN_METADATA_ENTRY_LABELS, MetadataEntry} from '../metadata/MetadataEntry'; -import {isCanonicalColumnLineageEntry, isCanonicalColumnSchemaEntry} from '../metadata/TableSchema'; +import {isCanonicalTableSchemaEntry} from '../metadata/TableSchema'; import {MetadataEntryFragment} from '../metadata/types/MetadataEntry.types'; import {titleForRun} from '../runs/RunUtils'; @@ -89,14 +89,14 @@ export const AssetEventMetadataEntriesTable = ({ const filteredRows = useMemo( () => - allRows - .filter((row) => !filter || row.entry.label.toLowerCase().includes(filter.toLowerCase())) - .filter( - (row) => - !HIDDEN_METADATA_ENTRY_LABELS.has(row.entry.label) && - !(isCanonicalColumnSchemaEntry(row.entry) && hideTableSchema) && - !isCanonicalColumnLineageEntry(row.entry), - ), + allRows.filter( + (row) => + !filter || + row.entry.label.toLowerCase().includes(filter.toLowerCase()) || + !HIDDEN_METADATA_ENTRY_LABELS.has(row.entry.label) || + !hideTableSchema || + !isCanonicalTableSchemaEntry(row.entry), + ), [allRows, filter, hideTableSchema], ); diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineage.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineage.tsx index 04306e03958b8..69fff663b3b46 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineage.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineage.tsx @@ -5,24 +5,17 @@ import { Colors, Icon, JoinedButtons, - MenuItem, - Suggest, TextInput, } from '@dagster-io/ui-components'; -import React, {useEffect, useMemo, useState} from 'react'; +import {useEffect, useMemo, useState} from 'react'; import styled from 'styled-components'; -import {AssetColumnLineageGraph} from './AssetColumnLineageGraph'; import {AssetNodeLineageGraph} from './AssetNodeLineageGraph'; import {LaunchAssetExecutionButton} from './LaunchAssetExecutionButton'; -import {asAssetKeyInput} from './asInput'; -import {useColumnLineageDataForAssets} from './lineage/useColumnLineageDataForAssets'; import {AssetLineageScope, AssetViewParams} from './types'; -import {GraphData, toGraphId} from '../asset-graph/Utils'; +import {GraphData} from '../asset-graph/Utils'; import {AssetGraphQueryItem, calculateGraphDistances} from '../asset-graph/useAssetGraphData'; import {AssetKeyInput} from '../graphql/types'; -import {useQueryPersistedState} from '../hooks/useQueryPersistedState'; -import {ClearButton} from '../ui/ClearButton'; export const AssetNodeLineage = ({ params, @@ -52,18 +45,6 @@ export const AssetNodeLineage = ({ const currentDepth = Math.max(1, Math.min(maxDepth, requestedDepth)); - const assetGraphKeys = useMemo( - () => Object.values(assetGraphData.nodes).map(asAssetKeyInput), - [assetGraphData], - ); - const columnLineageData = useColumnLineageDataForAssets(assetGraphKeys); - const columnLineage = columnLineageData[toGraphId(assetKey)]; - const [column, setColumn] = useQueryPersistedState({ - queryKey: 'column', - decode: (qs) => qs.column || null, - encode: (column) => ({column: column || undefined}), - }); - return ( setParams({...params, lineageDepth: depth})} max={maxDepth} /> - {columnLineage || column ? ( - <> - Column - setColumn(null)} - style={{marginTop: 5, marginRight: 4}} - > - - - ) : undefined, - }} - selectedItem={column} - items={Object.keys(columnLineage || {})} - noResults="No matching columns" - onItemSelect={setColumn} - inputValueRenderer={(item) => item} - itemPredicate={(query, item) => - item.toLocaleLowerCase().includes(query.toLocaleLowerCase()) - } - itemRenderer={(item, itemProps) => ( - itemProps.handleClick(e)} - text={item} - key={item} - /> - )} - /> - - ) : undefined}
{Object.values(assetGraphData.nodes).length > 1 ? ( )} - {column ? ( - - ) : ( - - )} + ); }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineageGraph.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineageGraph.tsx index 7d9df75ba23ab..13d99ed05388b 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineageGraph.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineageGraph.tsx @@ -1,9 +1,8 @@ import {Box, Spinner} from '@dagster-io/ui-components'; -import {useMemo, useRef, useState} from 'react'; +import {useEffect, useMemo, useRef, useState} from 'react'; import {useHistory} from 'react-router-dom'; import styled from 'styled-components'; -import {SVGSaveZoomLevel, useLastSavedZoomLevel} from './SavedZoomLevel'; import {assetDetailsPathForKey} from './assetDetailsPathForKey'; import {AssetKey, AssetViewParams} from './types'; import {AssetEdges} from '../asset-graph/AssetEdges'; @@ -12,13 +11,13 @@ import {AssetNode, AssetNodeContextMenuWrapper, AssetNodeMinimal} from '../asset import {ExpandedGroupNode, GroupOutline} from '../asset-graph/ExpandedGroupNode'; import {AssetNodeLink} from '../asset-graph/ForeignNode'; import {GraphData, GraphNode, groupIdForNode, toGraphId} from '../asset-graph/Utils'; -import {LayoutAssetGraphOptions} from '../asset-graph/layout'; import {DEFAULT_MAX_ZOOM, SVGViewport} from '../graph/SVGViewport'; import {useAssetLayout} from '../graph/asyncGraphLayout'; import {isNodeOffscreen} from '../graph/common'; import {AssetKeyInput} from '../graphql/types'; +import {getJSONForKey} from '../hooks/useStateWithStorage'; -const LINEAGE_GRAPH_OPTIONS: LayoutAssetGraphOptions = {direction: 'horizontal'}; +const LINEAGE_GRAPH_ZOOM_LEVEL = 'lineageGraphZoomLevel'; export const AssetNodeLineageGraph = ({ assetKey, @@ -43,7 +42,7 @@ export const AssetNodeLineageGraph = ({ const [highlighted, setHighlighted] = useState(null); - const {layout, loading} = useAssetLayout(assetGraphData, allGroups, LINEAGE_GRAPH_OPTIONS); + const {layout, loading} = useAssetLayout(assetGraphData, allGroups, 'horizontal'); const viewportEl = useRef(); const history = useHistory(); @@ -51,7 +50,13 @@ export const AssetNodeLineageGraph = ({ history.push(assetDetailsPathForKey(key, {...params, lineageScope: 'neighbors'})); }; - useLastSavedZoomLevel(viewportEl, layout, assetGraphId); + useEffect(() => { + if (viewportEl.current && layout) { + const lastZoomLevel = Number(getJSONForKey(LINEAGE_GRAPH_ZOOM_LEVEL)); + viewportEl.current.autocenter(false, lastZoomLevel); + viewportEl.current.focus(); + } + }, [viewportEl, layout, assetGraphId]); if (!layout || loading) { return ( @@ -108,7 +113,10 @@ export const AssetNodeLineageGraph = ({ .map((group) => ( @@ -166,6 +174,17 @@ export const AssetNodeLineageGraph = ({ ); }; +const SVGSaveZoomLevel = ({scale}: {scale: number}) => { + useEffect(() => { + try { + window.localStorage.setItem(LINEAGE_GRAPH_ZOOM_LEVEL, JSON.stringify(scale)); + } catch (err) { + // no-op + } + }, [scale]); + return <>; +}; + const SVGContainer = styled.svg` overflow: visible; border-radius: 0; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeOverview.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeOverview.tsx index 7fb5e537cf297..3d99f1881739b 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeOverview.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeOverview.tsx @@ -36,7 +36,6 @@ import {SimpleStakeholderAssetStatus} from './SimpleStakeholderAssetStatus'; import {UnderlyingOpsOrGraph} from './UnderlyingOpsOrGraph'; import {AssetChecksStatusSummary} from './asset-checks/AssetChecksStatusSummary'; import {assetDetailsPathForKey} from './assetDetailsPathForKey'; -import {buildConsolidatedColumnSchema} from './buildConsolidatedColumnSchema'; import {globalAssetGraphPathForAssetsAndDescendants} from './globalAssetGraphPathToString'; import {AssetKey} from './types'; import {AssetNodeDefinitionFragment} from './types/AssetNodeDefinition.types'; @@ -56,11 +55,7 @@ import {DagsterTypeSummary} from '../dagstertype/DagsterType'; import {AssetComputeKindTag} from '../graph/OpTags'; import {useStateWithStorage} from '../hooks/useStateWithStorage'; import {useLaunchPadHooks} from '../launchpad/LaunchpadHooksContext'; -import { - TableSchema, - TableSchemaLineageContext, - isCanonicalColumnLineageEntry, -} from '../metadata/TableSchema'; +import {TableSchema, isCanonicalTableSchemaEntry} from '../metadata/TableSchema'; import {RepositoryLink} from '../nav/RepositoryLink'; import {ScheduleOrSensorTag} from '../nav/ScheduleOrSensorTag'; import {useRepositoryLocationForAddress} from '../nav/useRepositoryLocationForAddress'; @@ -107,13 +102,35 @@ export const AssetNodeOverview = ({ return ; } - const {tableSchema, tableSchemaLoadTimestamp} = buildConsolidatedColumnSchema({ - materialization, - definition: assetNode, - definitionLoadTimestamp: assetNodeLoadTimestamp, - }); - - const columnSchema = materialization?.metadataEntries.find(isCanonicalColumnLineageEntry); + const materializationTableSchema = materialization?.metadataEntries.find( + isCanonicalTableSchemaEntry, + ); + const materializationTableSchemaLoadTimestamp = materialization + ? Number(materialization.timestamp) + : undefined; + const definitionTableSchema = assetNode?.metadataEntries.find(isCanonicalTableSchemaEntry); + const definitionTableSchemaLoadTimestamp = assetNodeLoadTimestamp; + + let tableSchema = materializationTableSchema ?? definitionTableSchema; + const tableSchemaLoadTimestamp = + materializationTableSchemaLoadTimestamp ?? definitionTableSchemaLoadTimestamp; + + // Merge the descriptions from the definition table schema with the materialization table schema + if (materializationTableSchema && definitionTableSchema) { + const definitionTableSchemaColumnDescriptionsByName = Object.fromEntries( + definitionTableSchema.schema.columns.map((column) => [column.name, column.description]), + ); + const mergedColumns = materializationTableSchema.schema.columns.map((column) => { + const description = + definitionTableSchemaColumnDescriptionsByName[column.name] || column.description; + return {...column, description}; + }); + + tableSchema = { + ...materializationTableSchema, + schema: {...materializationTableSchema.schema, columns: mergedColumns}, + }; + } const renderStatusSection = () => ( @@ -394,14 +411,10 @@ export const AssetNodeOverview = ({ {tableSchema && ( - - - + )} diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx index 01033017d8822..219673952c2e2 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx @@ -366,7 +366,7 @@ function getQueryForVisibleAssets( return {query: `+"${token}"+`, requestedDepth: 1}; } if (view === 'lineage') { - const defaultDepth = 1; + const defaultDepth = lineageScope === 'neighbors' ? 2 : 5; const requestedDepth = Number(lineageDepth) || defaultDepth; const depthStr = '+'.repeat(requestedDepth); diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/LastMaterializationMetadata.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/LastMaterializationMetadata.tsx index 63263400d7b5a..6c3c9559b59c6 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/LastMaterializationMetadata.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/LastMaterializationMetadata.tsx @@ -15,7 +15,6 @@ import {Timestamp} from '../app/time/Timestamp'; import {LiveDataForNode, isHiddenAssetGroupJob} from '../asset-graph/Utils'; import {AssetKeyInput} from '../graphql/types'; import {MetadataEntry} from '../metadata/MetadataEntry'; -import {isCanonicalColumnLineageEntry} from '../metadata/TableSchema'; import {Description} from '../pipelines/Description'; import {PipelineReference} from '../pipelines/PipelineReference'; import {linkToRunEvent, titleForRun} from '../runs/RunUtils'; @@ -157,20 +156,18 @@ export const LatestMaterializationMetadata = ({ ) : null} - {latestEvent.metadataEntries - .filter((entry) => !isCanonicalColumnLineageEntry(entry)) - .map((entry) => ( - - {entry.label} - - - - - ))} + {latestEvent.metadataEntries.map((entry) => ( + + {entry.label} + + + + + ))} ) : ( diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/SavedZoomLevel.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/SavedZoomLevel.tsx deleted file mode 100644 index 3d75adb38f5c0..0000000000000 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/SavedZoomLevel.tsx +++ /dev/null @@ -1,31 +0,0 @@ -import {MutableRefObject, useEffect} from 'react'; - -import {SVGViewport} from '../graph/SVGViewport'; -import {getJSONForKey} from '../hooks/useStateWithStorage'; - -const LINEAGE_GRAPH_ZOOM_LEVEL = 'lineageGraphZoomLevel'; - -export const SVGSaveZoomLevel = ({scale}: {scale: number}) => { - useEffect(() => { - try { - window.localStorage.setItem(LINEAGE_GRAPH_ZOOM_LEVEL, JSON.stringify(scale)); - } catch (err) { - // no-op - } - }, [scale]); - return <>; -}; - -export function useLastSavedZoomLevel( - viewportEl: MutableRefObject, - layout: import('../asset-graph/layout').AssetGraphLayout | null, - graphFocusChangeKey: string, -) { - useEffect(() => { - if (viewportEl.current && layout) { - const lastZoomLevel = Number(getJSONForKey(LINEAGE_GRAPH_ZOOM_LEVEL)); - viewportEl.current.autocenter(false, lastZoomLevel); - viewportEl.current.focus(); - } - }, [viewportEl, layout, graphFocusChangeKey]); -} diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/buildConsolidatedColumnSchema.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/buildConsolidatedColumnSchema.tsx deleted file mode 100644 index c4081b0699bb0..0000000000000 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/buildConsolidatedColumnSchema.tsx +++ /dev/null @@ -1,51 +0,0 @@ -// eslint-disable-next-line no-restricted-imports - -import {AssetColumnLineageQuery} from './lineage/types/useColumnLineageDataForAssets.types'; -import {isCanonicalColumnSchemaEntry} from '../metadata/TableSchema'; - -type AssetDefinitionWithMetadata = AssetColumnLineageQuery['assetNodes'][0]; - -/** - * This helper pulls the `columns` metadata entry from the most recent materialization - * and the asset definition, blending the two together to produce the most current - * representation. (Sometimes descriptions are only in the definition-time version) - */ -export function buildConsolidatedColumnSchema({ - materialization, - definition, - definitionLoadTimestamp, -}: { - materialization: - | Pick - | undefined; - definition: Pick | undefined; - definitionLoadTimestamp: number | undefined; -}) { - const materializationTableSchema = materialization?.metadataEntries.find( - isCanonicalColumnSchemaEntry, - ); - const materializationTimestamp = materialization ? Number(materialization.timestamp) : undefined; - const definitionTableSchema = definition?.metadataEntries.find(isCanonicalColumnSchemaEntry); - - let tableSchema = materializationTableSchema ?? definitionTableSchema; - const tableSchemaLoadTimestamp = materializationTimestamp ?? definitionLoadTimestamp; - - // Merge the descriptions from the definition table schema with the materialization table schema - if (materializationTableSchema && definitionTableSchema) { - const definitionTableSchemaColumnDescriptionsByName = Object.fromEntries( - definitionTableSchema.schema.columns.map((column) => [column.name, column.description]), - ); - const mergedColumns = materializationTableSchema.schema.columns.map((column) => { - const description = - definitionTableSchemaColumnDescriptionsByName[column.name] || column.description; - return {...column, description}; - }); - - tableSchema = { - ...materializationTableSchema, - schema: {...materializationTableSchema.schema, columns: mergedColumns}, - }; - } - console.log(tableSchema); - return {tableSchema, tableSchemaLoadTimestamp}; -} diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/lineage/types/useColumnLineageDataForAssets.types.ts b/js_modules/dagster-ui/packages/ui-core/src/assets/lineage/types/useColumnLineageDataForAssets.types.ts deleted file mode 100644 index e9db8c746bf02..0000000000000 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/lineage/types/useColumnLineageDataForAssets.types.ts +++ /dev/null @@ -1,82 +0,0 @@ -// Generated GraphQL types, do not edit manually. - -import * as Types from '../../../graphql/types'; - -export type AssetColumnLineageQueryVariables = Types.Exact<{ - assetKeys: Array | Types.AssetKeyInput; -}>; - -export type AssetColumnLineageQuery = { - __typename: 'Query'; - assetNodes: Array<{ - __typename: 'AssetNode'; - id: string; - assetKey: {__typename: 'AssetKey'; path: Array}; - metadataEntries: Array< - | {__typename: 'AssetMetadataEntry'; label: string} - | {__typename: 'BoolMetadataEntry'; label: string} - | {__typename: 'FloatMetadataEntry'; label: string} - | {__typename: 'IntMetadataEntry'; label: string} - | {__typename: 'JobMetadataEntry'; label: string} - | {__typename: 'JsonMetadataEntry'; label: string} - | {__typename: 'MarkdownMetadataEntry'; label: string} - | {__typename: 'NotebookMetadataEntry'; label: string} - | {__typename: 'NullMetadataEntry'; label: string} - | {__typename: 'PathMetadataEntry'; label: string} - | {__typename: 'PipelineRunMetadataEntry'; label: string} - | {__typename: 'PythonArtifactMetadataEntry'; label: string} - | {__typename: 'TableMetadataEntry'; label: string} - | { - __typename: 'TableSchemaMetadataEntry'; - label: string; - schema: { - __typename: 'TableSchema'; - columns: Array<{ - __typename: 'TableColumn'; - name: string; - type: string; - description: string | null; - }>; - }; - } - | {__typename: 'TextMetadataEntry'; label: string} - | {__typename: 'TimestampMetadataEntry'; label: string} - | {__typename: 'UrlMetadataEntry'; label: string} - >; - assetMaterializations: Array<{ - __typename: 'MaterializationEvent'; - timestamp: string; - metadataEntries: Array< - | {__typename: 'AssetMetadataEntry'; label: string} - | {__typename: 'BoolMetadataEntry'; label: string} - | {__typename: 'FloatMetadataEntry'; label: string} - | {__typename: 'IntMetadataEntry'; label: string} - | {__typename: 'JobMetadataEntry'; label: string} - | {__typename: 'JsonMetadataEntry'; jsonString: string; label: string} - | {__typename: 'MarkdownMetadataEntry'; label: string} - | {__typename: 'NotebookMetadataEntry'; label: string} - | {__typename: 'NullMetadataEntry'; label: string} - | {__typename: 'PathMetadataEntry'; label: string} - | {__typename: 'PipelineRunMetadataEntry'; label: string} - | {__typename: 'PythonArtifactMetadataEntry'; label: string} - | {__typename: 'TableMetadataEntry'; label: string} - | { - __typename: 'TableSchemaMetadataEntry'; - label: string; - schema: { - __typename: 'TableSchema'; - columns: Array<{ - __typename: 'TableColumn'; - name: string; - type: string; - description: string | null; - }>; - }; - } - | {__typename: 'TextMetadataEntry'; label: string} - | {__typename: 'TimestampMetadataEntry'; label: string} - | {__typename: 'UrlMetadataEntry'; label: string} - >; - }>; - }>; -}; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/lineage/useColumnLineageDataForAssets.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/lineage/useColumnLineageDataForAssets.tsx deleted file mode 100644 index 5140b7a95af8d..0000000000000 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/lineage/useColumnLineageDataForAssets.tsx +++ /dev/null @@ -1,158 +0,0 @@ -import {gql, useApolloClient} from '@apollo/client'; -import React, {useMemo, useRef, useState} from 'react'; - -import { - AssetColumnLineageQuery, - AssetColumnLineageQueryVariables, -} from './types/useColumnLineageDataForAssets.types'; -import {toGraphId} from '../../asset-graph/Utils'; -import {AssetKeyInput} from '../../graphql/types'; -import {isCanonicalColumnLineageEntry} from '../../metadata/TableSchema'; -import {buildConsolidatedColumnSchema} from '../buildConsolidatedColumnSchema'; - -export type AssetColumnLineageServer = { - [column: string]: { - // Note: This is [["key_part_1", "key_part_2"]] but the outer array - // only contains one item, it's a serialization odditiy. - upstream_asset_key: string[][]; - upstream_column_name: string; - }[]; -}; - -export type AssetColumnLineageLocalColumn = { - name: string; - type: string | null; - description: string | null; - asOf: string | undefined; // materialization timestamp - upstream: { - assetKey: AssetKeyInput; - columnName: string; - }[]; -}; - -export type AssetColumnLineageLocal = { - [column: string]: AssetColumnLineageLocalColumn; -}; - -export type AssetColumnLineages = {[graphId: string]: AssetColumnLineageLocal | undefined}; - -/** - * The column definitions and the column lineage are in two separate metadata entries, - * and the definitions may be specified in definition-time or materialization-time metadata. - * Parse them both and combine the results into a single representation of asset columns - * that is easier for the rest of the front-end to use. - */ -const getColumnLineage = ( - asset: AssetColumnLineageQuery['assetNodes'][0], -): AssetColumnLineageLocal | undefined => { - const materialization = asset.assetMaterializations[0]; - const lineageMetadata = materialization?.metadataEntries.find(isCanonicalColumnLineageEntry); - if (!lineageMetadata) { - return undefined; - } - - const {tableSchema} = buildConsolidatedColumnSchema({ - materialization, - definition: asset, - definitionLoadTimestamp: undefined, - }); - - const lineageParsed: AssetColumnLineageServer = JSON.parse(lineageMetadata.jsonString); - const schemaParsed = tableSchema?.schema - ? Object.fromEntries(tableSchema.schema.columns.map((col) => [col.name, col])) - : {}; - - return Object.fromEntries( - Object.entries(lineageParsed).map(([column, m]) => [ - column, - { - name: column, - asOf: materialization?.timestamp, - type: schemaParsed[column]?.type || null, - description: schemaParsed[column]?.description || null, - upstream: m.map((u) => ({ - assetKey: {path: u.upstream_asset_key[0]!}, - columnName: u.upstream_column_name, - })), - }, - ]), - ); -}; - -export function useColumnLineageDataForAssets(assetKeys: AssetKeyInput[]) { - const [loaded, setLoaded] = useState({}); - const client = useApolloClient(); - const fetching = useRef(false); - const missing = useMemo( - () => assetKeys.filter((a) => !loaded[toGraphId(a)]), - [assetKeys, loaded], - ); - - React.useEffect(() => { - const fetch = async () => { - fetching.current = true; - const {data} = await client.query({ - query: ASSET_COLUMN_LINEAGE_QUERY, - variables: {assetKeys: missing}, - }); - fetching.current = false; - - setLoaded((loaded) => ({ - ...loaded, - ...Object.fromEntries( - data.assetNodes.map((n) => [toGraphId(n.assetKey), getColumnLineage(n)]), - ), - })); - }; - if (!fetching.current && missing.length) { - void fetch(); - } - }, [client, missing]); - - return loaded; -} - -const ASSET_COLUMN_LINEAGE_QUERY = gql` - query AssetColumnLineage($assetKeys: [AssetKeyInput!]!) { - assetNodes(loadMaterializations: true, assetKeys: $assetKeys) { - id - assetKey { - path - } - metadataEntries { - __typename - label - ... on TableSchemaMetadataEntry { - label - schema { - columns { - name - type - description - } - } - } - } - assetMaterializations(limit: 1) { - timestamp - metadataEntries { - __typename - label - ... on TableSchemaMetadataEntry { - label - schema { - columns { - name - type - description - } - } - } - ... on JsonMetadataEntry { - jsonString - } - } - } - } - } -`; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/types.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/types.tsx index b87c0bfaabdf1..b61cf71a59ed6 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/types.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/types.tsx @@ -21,5 +21,4 @@ export interface AssetViewParams { evaluation?: string; checkDetail?: string; default_range?: string; - column?: string; } diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/useColumnLineageLayout.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/useColumnLineageLayout.tsx deleted file mode 100644 index 376b60e6a069a..0000000000000 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/useColumnLineageLayout.tsx +++ /dev/null @@ -1,142 +0,0 @@ -import {useMemo} from 'react'; - -import {AssetColumnLineages} from './lineage/useColumnLineageDataForAssets'; -import {GraphData, groupIdForNode, toGraphId, tokenForAssetKey} from '../asset-graph/Utils'; -import {LayoutAssetGraphOptions} from '../asset-graph/layout'; -import {useAssetLayout} from '../graph/asyncGraphLayout'; - -const LINEAGE_GRAPH_COLUMN_LAYOUT_OPTIONS: LayoutAssetGraphOptions = { - direction: 'horizontal', - overrides: { - nodeHeight: 32, - nodesep: 0, - edgesep: 0, - clusterpaddingtop: 50, - groupPaddingBottom: -3, - groupPaddingTop: 68, - groupRendering: 'always', - }, -}; - -type Item = {assetGraphId: string; column: string; direction?: 'upstream' | 'downstream'}; - -export function toColumnGraphId(item: {assetGraphId: string; column: string}) { - return JSON.stringify({assetGraphId: item.assetGraphId, column: item.column}); -} -export function fromColumnGraphId(id: string) { - return JSON.parse(id) as {assetGraphId: string; column: string}; -} - -/** - * This function returns GraphData in which each `node` is a column of an asset and each `group` - * is an asset, essentially "zooming in" to the asset column level. This is a bit awkward but - * allows us to reuse the asset layout engine (and all it's caching, async dispatch, etc) for - * this view. - */ -export function useColumnLineageLayout( - assetGraphData: GraphData, - assetGraphId: string, - column: string, - columnLineageData: AssetColumnLineages, -) { - const {columnGraphData, groups} = useMemo(() => { - const columnGraphData: GraphData = { - nodes: {}, - downstream: {}, - upstream: {}, - }; - - const downstreams = buildReverseEdgeLookupTable(columnLineageData); - - const addEdge = (upstreamId: string, downstreamId: string) => { - columnGraphData.upstream[downstreamId] = columnGraphData.upstream[downstreamId] || {}; - columnGraphData.upstream[downstreamId]![upstreamId] = true; - columnGraphData.downstream[upstreamId] = columnGraphData.downstream[upstreamId] || {}; - columnGraphData.downstream[upstreamId]![downstreamId] = true; - }; - - const groups = new Set(); - const queue: Item[] = [{assetGraphId, column}]; - let item: Item | undefined; - - while ((item = queue.pop())) { - if (!item) { - continue; - } - const id = toColumnGraphId(item); - const {assetGraphId, column, direction} = item; - const assetNode = assetGraphData.nodes[assetGraphId]; - if (columnGraphData.nodes[id] || !assetNode) { - continue; // visited already - } - - const columnGraphNode = { - id, - assetKey: assetNode.assetKey, - definition: { - ...assetNode.definition, - groupName: `${tokenForAssetKey(assetNode.assetKey)}`, - }, - }; - columnGraphData.nodes[id] = columnGraphNode; - groups.add(groupIdForNode(columnGraphNode)); - - if (!direction || direction === 'upstream') { - const lineageForColumn = columnLineageData[assetGraphId]?.[column]; - for (const upstream of lineageForColumn?.upstream || []) { - const upstreamGraphId = toGraphId(upstream.assetKey); - const upstreamItem: Item = { - assetGraphId: upstreamGraphId, - column: upstream.columnName, - direction: 'upstream', - }; - if (assetGraphData.nodes[upstreamItem.assetGraphId]) { - queue.push(upstreamItem); - addEdge(toColumnGraphId(upstreamItem), id); - } - } - } - if (!direction || direction === 'downstream') { - for (const downstreamId of Object.keys(downstreams[id] || {})) { - const downstreamItem: Item = { - ...fromColumnGraphId(downstreamId), - direction: 'downstream', - }; - if (assetGraphData.nodes[downstreamItem.assetGraphId]) { - queue.push(downstreamItem); - addEdge(id, downstreamId); - } - } - } - } - - return {columnGraphData, groups: Array.from(groups)}; - }, [assetGraphData, column, columnLineageData, assetGraphId]); - - return useAssetLayout(columnGraphData, groups, LINEAGE_GRAPH_COLUMN_LAYOUT_OPTIONS); -} - -/** - * The column lineage data we get from asset metadata only gives us upstreams for each column. - * To efficiently build graph data we need both upstreams and downstreams for each column. - * This function visits every node and builds a downstreams lookup table. - */ -function buildReverseEdgeLookupTable(columnLineageData: AssetColumnLineages) { - const downstreams: {[id: string]: {[id: string]: true}} = {}; - - Object.entries(columnLineageData).forEach(([downstreamAssetGraphId, e]) => { - Object.entries(e || {}).forEach(([downstreamColumnName, {upstream}]) => { - const downstreamKey = toColumnGraphId({ - assetGraphId: downstreamAssetGraphId, - column: downstreamColumnName, - }); - for (const {assetKey, columnName} of upstream) { - const key = toColumnGraphId({assetGraphId: toGraphId(assetKey), column: columnName}); - downstreams[key] = downstreams[key] || {}; - downstreams[key]![downstreamKey] = true; - } - }); - }); - - return downstreams; -} diff --git a/js_modules/dagster-ui/packages/ui-core/src/graph/asyncGraphLayout.ts b/js_modules/dagster-ui/packages/ui-core/src/graph/asyncGraphLayout.ts index fa47bbe03099b..d7ec4bf980477 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/graph/asyncGraphLayout.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/graph/asyncGraphLayout.ts @@ -5,7 +5,12 @@ import {ILayoutOp, LayoutOpGraphOptions, OpGraphLayout, layoutOpGraph} from './l import {useFeatureFlags} from '../app/Flags'; import {asyncMemoize, indexedDBAsyncMemoize} from '../app/Util'; import {GraphData} from '../asset-graph/Utils'; -import {AssetGraphLayout, LayoutAssetGraphOptions, layoutAssetGraph} from '../asset-graph/layout'; +import { + AssetGraphLayout, + AssetLayoutDirection, + LayoutAssetGraphOptions, + layoutAssetGraph, +} from '../asset-graph/layout'; const ASYNC_LAYOUT_SOLID_COUNT = 50; @@ -181,13 +186,14 @@ export function useOpLayout(ops: ILayoutOp[], parentOp?: ILayoutOp) { export function useAssetLayout( _graphData: GraphData, expandedGroups: string[], - opts: LayoutAssetGraphOptions, + direction: AssetLayoutDirection, ) { const [state, dispatch] = useReducer(reducer, initialState); const flags = useFeatureFlags(); const graphData = useMemo(() => ({..._graphData, expandedGroups}), [expandedGroups, _graphData]); + const opts = useMemo(() => ({direction}), [direction]); const cacheKey = _assetLayoutCacheKey(graphData, opts); const nodeCount = Object.keys(graphData.nodes).length; const runAsync = nodeCount >= ASYNC_LAYOUT_SOLID_COUNT; diff --git a/js_modules/dagster-ui/packages/ui-core/src/metadata/MetadataEntry.tsx b/js_modules/dagster-ui/packages/ui-core/src/metadata/MetadataEntry.tsx index a50b1a299139a..4f44ea1b94c79 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/metadata/MetadataEntry.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/metadata/MetadataEntry.tsx @@ -223,7 +223,6 @@ export const MetadataEntry = ({ ) : ( JSON.stringify(entry.schema, null, 2)} content={() => ( React.ReactNode; copyContent: () => string; }) => { @@ -379,7 +377,7 @@ const MetadataEntryModalAction = (props: { setOpen(true)}>{props.children} setOpen(false)} isOpen={open} diff --git a/js_modules/dagster-ui/packages/ui-core/src/metadata/TableSchema.tsx b/js_modules/dagster-ui/packages/ui-core/src/metadata/TableSchema.tsx index 839e83cd86460..7982db4f315e0 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/metadata/TableSchema.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/metadata/TableSchema.tsx @@ -11,20 +11,12 @@ import { Tooltip, } from '@dagster-io/ui-components'; import {Spacing} from '@dagster-io/ui-components/src/components/types'; -import {createContext, useContext, useState} from 'react'; +import {useState} from 'react'; import {TableSchemaFragment} from './types/TableSchema.types'; import {Timestamp} from '../app/time/Timestamp'; import {StyledTableWithHeader} from '../assets/AssetEventMetadataEntriesTable'; -import {assetDetailsPathForKey} from '../assets/assetDetailsPathForKey'; -import { - AssetKeyInput, - JsonMetadataEntry, - MaterializationEvent, - TableSchemaMetadataEntry, -} from '../graphql/types'; -import {Description} from '../pipelines/Description'; -import {AnchorButton} from '../ui/AnchorButton'; +import {MaterializationEvent, TableSchemaMetadataEntry} from '../graphql/types'; type ITableSchema = TableSchemaFragment; @@ -36,26 +28,16 @@ interface ITableSchemaProps { itemHorizontalPadding?: Spacing; } -export const isCanonicalColumnSchemaEntry = ( +export const isCanonicalTableSchemaEntry = ( m: Pick, ): m is TableSchemaMetadataEntry => m.__typename === 'TableSchemaMetadataEntry' && m.label === 'dagster/column_schema'; -export const isCanonicalColumnLineageEntry = ( - m: Pick, -): m is JsonMetadataEntry => m.__typename === 'JsonMetadataEntry' && m.label === 'lineage'; - -export const TableSchemaLineageContext = createContext<{assetKey: AssetKeyInput | null}>({ - assetKey: null, -}); - export const TableSchema = ({ schema, schemaLoadTimestamp, itemHorizontalPadding, }: ITableSchemaProps) => { - const {assetKey} = useContext(TableSchemaLineageContext); - const multiColumnConstraints = schema.constraints?.other || []; const [filter, setFilter] = useState(''); const rows = schema.columns.filter( @@ -94,7 +76,6 @@ export const TableSchema = ({ Column name Type Description - {assetKey ? : undefined} @@ -104,29 +85,14 @@ export const TableSchema = ({ {column.name} - + {!column.constraints.nullable && NonNullableTag} {column.constraints.unique && UniqueTag} {column.constraints.other.map((constraint, i) => ( ))} - - - - {assetKey ? ( - - - } - to={assetDetailsPathForKey(assetKey, { - view: 'lineage', - column: column.name, - })} - /> - - - ) : undefined} + {column.description} ))} {rows.length === 0 && ( @@ -142,7 +108,7 @@ export const TableSchema = ({ ); }; -export const iconForColumnType = (type: string): IconName | null => { +const iconForType = (type: string): IconName | null => { const lower = type.toLowerCase(); if (lower.includes('bool')) { return 'datatype_bool'; @@ -162,14 +128,12 @@ export const iconForColumnType = (type: string): IconName | null => { return null; }; -export const TypeTag = ({type = ''}: {type: string}) => { +const TypeTag = ({type = '', icon}: {type: string; icon: IconName | null}) => { if (type.trim().replace(/\?/g, '').length === 0) { // Do not render type '' or '?' or any other empty value. return ; } - const icon = iconForColumnType(type); - return ( diff --git a/js_modules/dagster-ui/patches/dagre+0.8.5.patch b/js_modules/dagster-ui/patches/dagre+0.8.5.patch new file mode 100644 index 0000000000000..fab38165260b3 --- /dev/null +++ b/js_modules/dagster-ui/patches/dagre+0.8.5.patch @@ -0,0 +1,15 @@ +diff --git a/node_modules/dagre/lib/order/index.js b/node_modules/dagre/lib/order/index.js +index 4ac2d9f..a2182fe 100644 +--- a/node_modules/dagre/lib/order/index.js ++++ b/node_modules/dagre/lib/order/index.js +@@ -73,7 +73,9 @@ function sweepLayerGraphs(layerGraphs, biasRight) { + function assignOrder(g, layering) { + _.forEach(layering, function(layer) { + _.forEach(layer, function(v, i) { +- g.node(v).order = i; ++ try { ++ g.node(v).order = i; ++ } catch (e) {} + }); + }); + } diff --git a/js_modules/dagster-ui/yarn.lock b/js_modules/dagster-ui/yarn.lock index fb58b04e57528..f44fbb20646dc 100644 --- a/js_modules/dagster-ui/yarn.lock +++ b/js_modules/dagster-ui/yarn.lock @@ -2598,7 +2598,7 @@ __metadata: codemirror: "npm:^5.65.2" color: "npm:^3.0.0" cronstrue: "npm:^1.84.0" - dagre: "dagster-io/dagre#0.8.5" + dagre: "npm:^0.8.5" date-fns: "npm:^2.28.0" dayjs: "npm:^1.11.7" deepmerge: "npm:^4.2.2" @@ -11528,13 +11528,13 @@ __metadata: languageName: node linkType: hard -"dagre@dagster-io/dagre#0.8.5": +"dagre@npm:^0.8.5": version: 0.8.5 - resolution: "dagre@https://github.com/dagster-io/dagre.git#commit=c2a1821cc7f8a220e819461b82b6ddbf48189100" + resolution: "dagre@npm:0.8.5" dependencies: graphlib: "npm:^2.1.8" lodash: "npm:^4.17.15" - checksum: 6a94d8d9b1c3132b406b5921fd2bbd1a207c78fc1048216787fec68fad1d96f649cb084d1ae576f5456532a4275f22a113d140265f05937db60e5918a25adac5 + checksum: f39899e29e9090581d67177ef6e2dd3ca5d7f764fbf3de81758d879bba66fee6fd8802d41d0c5d3d9a0563b334e99e1454a8d6ab4ce17e8e4f50836a3a403fdd languageName: node linkType: hard