diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetColumnsNode.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetColumnsNode.tsx index 8e748fcc6ae2f..96db8139d59c9 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetColumnsNode.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetColumnsNode.tsx @@ -1,4 +1,4 @@ -import {Box, Caption, Colors, StyledTag, Tooltip} from '@dagster-io/ui-components'; +import {Box, Caption, Colors, Icon, StyledTag, Tooltip} from '@dagster-io/ui-components'; import {Link} from 'react-router-dom'; import styled from 'styled-components'; @@ -9,18 +9,22 @@ import { AssetNodeContainer, } from './AssetNode'; import {AssetNodeFragment} from './types/AssetNode.types'; +import {Timestamp} from '../app/time/Timestamp'; import {assetDetailsPathForKey} from '../assets/assetDetailsPathForKey'; -import {AssetColumnLineageLocal} from '../assets/lineage/useColumnLineageDataForAssets'; +import {AssetColumnLineageLocalColumn} from '../assets/lineage/useColumnLineageDataForAssets'; +import {AssetComputeKindTag} from '../graph/OpTags'; import {AssetKeyInput} from '../graphql/types'; -import {TypeTag} from '../metadata/TableSchema'; +import {iconForColumnType} from '../metadata/TableSchema'; export const AssetColumnsGroupNode = ({ selected, definition, height, + asOf, }: { selected: boolean; definition: AssetNodeFragment; + asOf: string | undefined; height: number; }) => { return ( @@ -30,7 +34,15 @@ export const AssetColumnsGroupNode = ({ + + {asOf ? ( + + + + ) : undefined} + + ); @@ -42,9 +54,11 @@ export const AssetColumnNode = ({ selected, }: { assetKey: AssetKeyInput; - column: AssetColumnLineageLocal['']; + column: AssetColumnLineageLocalColumn; selected: boolean; }) => { + const icon = iconForColumnType(column.type ?? ''); + return ( @@ -52,10 +66,10 @@ export const AssetColumnNode = ({ to={assetDetailsPathForKey(assetKey, {view: 'lineage', column: column.name})} $selected={selected} > + {icon ? : } {column.name} - @@ -65,11 +79,10 @@ export const AssetColumnNode = ({ const ColumnLink = styled(Link)<{$selected: boolean}>` height: 28px; margin: 2px 0; - padding-left: 8px; + padding-left: 2px; padding-right: 4px; display: flex; gap: 4px; - justify-content: space-between; align-items: center; transition: background 100ms linear; border-radius: 8px; diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/SidebarAssetInfo.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/SidebarAssetInfo.tsx index 3ce4087a9cbb3..c6644d6256854 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/SidebarAssetInfo.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/SidebarAssetInfo.tsx @@ -32,6 +32,7 @@ import { import {DagsterTypeSummary} from '../dagstertype/DagsterType'; import {DagsterTypeFragment} from '../dagstertype/types/DagsterType.types'; import {METADATA_ENTRY_FRAGMENT} from '../metadata/MetadataEntry'; +import {TableSchemaLineageContext} from '../metadata/TableSchema'; import {Description} from '../pipelines/Description'; import {SidebarSection, SidebarTitle} from '../pipelines/SidebarComponents'; import {ResourceContainer, ResourceHeader} from '../pipelines/SidebarOpHelpers'; @@ -159,7 +160,12 @@ export const SidebarAssetInfo = ({graphNode}: {graphNode: GraphNode}) => { {assetMetadata.length > 0 && ( - + + + )} diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/layout.ts b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/layout.ts index 1f8240b4b4bfc..66500fd93f20d 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/layout.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/layout.ts @@ -47,6 +47,10 @@ export type LayoutAssetGraphConfig = dagre.GraphLabel & { groupPaddingTop: number; groupPaddingBottom: number; groupRendering: 'if-varied' | 'always'; + + /** Supported in Dagre, just not documented. Additional spacing between group nodes */ + clusterpaddingtop?: number; + clusterpaddingbottom?: number; }; export type LayoutAssetGraphOptions = { diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetColumnLineageGraph.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetColumnLineageGraph.tsx index 29408889fc719..613356af3971f 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetColumnLineageGraph.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetColumnLineageGraph.tsx @@ -75,6 +75,9 @@ export const AssetColumnLineageGraph = ({ node: graphNode!, }; + const cols = columnLineageData[groupAssetGraphId] || {}; + const colsAsOf = Object.values(cols)[0]?.asOf; + return ( @@ -117,6 +121,7 @@ export const AssetColumnLineageGraph = ({ description: 'Not found in column metadata', type: null, upstream: [], + asOf: undefined, }; return ( diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeOverview.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeOverview.tsx index 92c4a3c4217ee..7fb5e537cf297 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeOverview.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeOverview.tsx @@ -56,7 +56,11 @@ import {DagsterTypeSummary} from '../dagstertype/DagsterType'; import {AssetComputeKindTag} from '../graph/OpTags'; import {useStateWithStorage} from '../hooks/useStateWithStorage'; import {useLaunchPadHooks} from '../launchpad/LaunchpadHooksContext'; -import {TableSchema, isCanonicalColumnLineageEntry} from '../metadata/TableSchema'; +import { + TableSchema, + TableSchemaLineageContext, + isCanonicalColumnLineageEntry, +} from '../metadata/TableSchema'; import {RepositoryLink} from '../nav/RepositoryLink'; import {ScheduleOrSensorTag} from '../nav/ScheduleOrSensorTag'; import {useRepositoryLocationForAddress} from '../nav/useRepositoryLocationForAddress'; @@ -390,11 +394,14 @@ export const AssetNodeOverview = ({ {tableSchema && ( - + + + )} diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx index 219673952c2e2..01033017d8822 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx @@ -366,7 +366,7 @@ function getQueryForVisibleAssets( return {query: `+"${token}"+`, requestedDepth: 1}; } if (view === 'lineage') { - const defaultDepth = lineageScope === 'neighbors' ? 2 : 5; + const defaultDepth = 1; const requestedDepth = Number(lineageDepth) || defaultDepth; const depthStr = '+'.repeat(requestedDepth); diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/buildConsolidatedColumnSchema.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/buildConsolidatedColumnSchema.tsx index 7298f8575d0d8..c4081b0699bb0 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/buildConsolidatedColumnSchema.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/buildConsolidatedColumnSchema.tsx @@ -1,9 +1,10 @@ // eslint-disable-next-line no-restricted-imports -import {AssetNodeDefinitionFragment} from './types/AssetNodeDefinition.types'; -import {AssetMaterializationFragment} from './types/useRecentAssetEvents.types'; +import {AssetColumnLineageQuery} from './lineage/types/useColumnLineageDataForAssets.types'; import {isCanonicalColumnSchemaEntry} from '../metadata/TableSchema'; +type AssetDefinitionWithMetadata = AssetColumnLineageQuery['assetNodes'][0]; + /** * This helper pulls the `columns` metadata entry from the most recent materialization * and the asset definition, blending the two together to produce the most current @@ -14,21 +15,20 @@ export function buildConsolidatedColumnSchema({ definition, definitionLoadTimestamp, }: { - materialization: Pick | undefined; - definition: AssetNodeDefinitionFragment | undefined; + materialization: + | Pick + | undefined; + definition: Pick | undefined; definitionLoadTimestamp: number | undefined; }) { const materializationTableSchema = materialization?.metadataEntries.find( isCanonicalColumnSchemaEntry, ); - const materializationTableSchemaLoadTimestamp = materialization - ? Number(materialization.timestamp) - : undefined; + const materializationTimestamp = materialization ? Number(materialization.timestamp) : undefined; const definitionTableSchema = definition?.metadataEntries.find(isCanonicalColumnSchemaEntry); let tableSchema = materializationTableSchema ?? definitionTableSchema; - const tableSchemaLoadTimestamp = - materializationTableSchemaLoadTimestamp ?? definitionLoadTimestamp; + const tableSchemaLoadTimestamp = materializationTimestamp ?? definitionLoadTimestamp; // Merge the descriptions from the definition table schema with the materialization table schema if (materializationTableSchema && definitionTableSchema) { @@ -46,6 +46,6 @@ export function buildConsolidatedColumnSchema({ schema: {...materializationTableSchema.schema, columns: mergedColumns}, }; } - + console.log(tableSchema); return {tableSchema, tableSchemaLoadTimestamp}; } diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/lineage/useColumnLineageDataForAssets.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/lineage/useColumnLineageDataForAssets.tsx index 22acd7ecf50d9..4d35907547d81 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/lineage/useColumnLineageDataForAssets.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/lineage/useColumnLineageDataForAssets.tsx @@ -19,16 +19,19 @@ export type AssetColumnLineageServer = { }[]; }; +export type AssetColumnLineageLocalColumn = { + name: string; + type: string | null; + description: string | null; + asOf: string | undefined; // materialization timestamp + upstream: { + assetKey: AssetKeyInput; + columnName: string; + }[]; +}; + export type AssetColumnLineageLocal = { - [column: string]: { - name: string; - type: string | null; - description: string | null; - upstream: { - assetKey: AssetKeyInput; - columnName: string; - }[]; - }; + [column: string]: AssetColumnLineageLocalColumn; }; export type AssetColumnLineages = {[graphId: string]: AssetColumnLineageLocal | undefined}; @@ -64,6 +67,7 @@ const getColumnLineage = ( column, { name: column, + asOf: materialization?.timestamp, type: schemaParsed[column]?.type || null, description: schemaParsed[column]?.description || null, upstream: m.map((u) => ({ diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/useColumnLineageLayout.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/useColumnLineageLayout.tsx index 084cb9267abe0..376b60e6a069a 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/useColumnLineageLayout.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/useColumnLineageLayout.tsx @@ -11,7 +11,8 @@ const LINEAGE_GRAPH_COLUMN_LAYOUT_OPTIONS: LayoutAssetGraphOptions = { nodeHeight: 32, nodesep: 0, edgesep: 0, - groupPaddingBottom: -5, + clusterpaddingtop: 50, + groupPaddingBottom: -3, groupPaddingTop: 68, groupRendering: 'always', }, diff --git a/js_modules/dagster-ui/packages/ui-core/src/metadata/MetadataEntry.tsx b/js_modules/dagster-ui/packages/ui-core/src/metadata/MetadataEntry.tsx index 4f44ea1b94c79..a50b1a299139a 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/metadata/MetadataEntry.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/metadata/MetadataEntry.tsx @@ -223,6 +223,7 @@ export const MetadataEntry = ({ ) : ( JSON.stringify(entry.schema, null, 2)} content={() => ( React.ReactNode; copyContent: () => string; }) => { @@ -377,7 +379,7 @@ const MetadataEntryModalAction = (props: { setOpen(true)}>{props.children} setOpen(false)} isOpen={open} diff --git a/js_modules/dagster-ui/packages/ui-core/src/metadata/TableSchema.tsx b/js_modules/dagster-ui/packages/ui-core/src/metadata/TableSchema.tsx index 5dd50842e8f48..839e83cd86460 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/metadata/TableSchema.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/metadata/TableSchema.tsx @@ -11,7 +11,7 @@ import { Tooltip, } from '@dagster-io/ui-components'; import {Spacing} from '@dagster-io/ui-components/src/components/types'; -import {useState} from 'react'; +import {createContext, useContext, useState} from 'react'; import {TableSchemaFragment} from './types/TableSchema.types'; import {Timestamp} from '../app/time/Timestamp'; @@ -23,6 +23,7 @@ import { MaterializationEvent, TableSchemaMetadataEntry, } from '../graphql/types'; +import {Description} from '../pipelines/Description'; import {AnchorButton} from '../ui/AnchorButton'; type ITableSchema = TableSchemaFragment; @@ -33,7 +34,6 @@ interface ITableSchemaProps { schema: ITableSchema; schemaLoadTimestamp?: number | undefined; itemHorizontalPadding?: Spacing; - assetKeyForColumnSchema?: AssetKeyInput; } export const isCanonicalColumnSchemaEntry = ( @@ -45,12 +45,17 @@ export const isCanonicalColumnLineageEntry = ( m: Pick, ): m is JsonMetadataEntry => m.__typename === 'JsonMetadataEntry' && m.label === 'lineage'; +export const TableSchemaLineageContext = createContext<{assetKey: AssetKeyInput | null}>({ + assetKey: null, +}); + export const TableSchema = ({ schema, schemaLoadTimestamp, itemHorizontalPadding, - assetKeyForColumnSchema, }: ITableSchemaProps) => { + const {assetKey} = useContext(TableSchemaLineageContext); + const multiColumnConstraints = schema.constraints?.other || []; const [filter, setFilter] = useState(''); const rows = schema.columns.filter( @@ -89,7 +94,7 @@ export const TableSchema = ({ Column name Type Description - {assetKeyForColumnSchema ? : undefined} + {assetKey ? : undefined} @@ -106,13 +111,15 @@ export const TableSchema = ({ ))} - {column.description} - {assetKeyForColumnSchema ? ( + + + + {assetKey ? ( } - to={assetDetailsPathForKey(assetKeyForColumnSchema, { + to={assetDetailsPathForKey(assetKey, { view: 'lineage', column: column.name, })} @@ -135,7 +142,7 @@ export const TableSchema = ({ ); }; -const iconForType = (type: string): IconName | null => { +export const iconForColumnType = (type: string): IconName | null => { const lower = type.toLowerCase(); if (lower.includes('bool')) { return 'datatype_bool'; @@ -161,7 +168,7 @@ export const TypeTag = ({type = ''}: {type: string}) => { return ; } - const icon = iconForType(type); + const icon = iconForColumnType(type); return (