diff --git a/js_modules/dagster-ui/packages/ui-components/src/components/RefreshableCountdown.tsx b/js_modules/dagster-ui/packages/ui-components/src/components/RefreshableCountdown.tsx index 594e3a89b4e00..841c1866961ce 100644 --- a/js_modules/dagster-ui/packages/ui-components/src/components/RefreshableCountdown.tsx +++ b/js_modules/dagster-ui/packages/ui-components/src/components/RefreshableCountdown.tsx @@ -9,7 +9,7 @@ import {secondsToCountdownTime} from './secondsToCountdownTime'; interface Props { refreshing: boolean; - seconds: number; + seconds?: number; onRefresh: () => void; dataDescription?: string; } @@ -21,7 +21,11 @@ export const RefreshableCountdown = (props: Props) => { - {refreshing ? `Refreshing ${dataDescription}…` : secondsToCountdownTime(seconds)} + {refreshing + ? `Refreshing ${dataDescription}…` + : seconds === undefined + ? null + : secondsToCountdownTime(seconds)} Refresh now} position="top"> diff --git a/js_modules/dagster-ui/packages/ui-core/.storybook/main.js b/js_modules/dagster-ui/packages/ui-core/.storybook/main.js index a66a37c84cd99..eb09a3db70581 100644 --- a/js_modules/dagster-ui/packages/ui-core/.storybook/main.js +++ b/js_modules/dagster-ui/packages/ui-core/.storybook/main.js @@ -56,6 +56,10 @@ const config = { docs: { autodocs: true, }, + env: (config) => ({ + ...config, + STORYBOOK: true, + }), }; export default config; diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-data/AssetLiveDataProvider.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-data/AssetLiveDataProvider.tsx index cc476ab5ff91d..7152946dea078 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-data/AssetLiveDataProvider.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-data/AssetLiveDataProvider.tsx @@ -1,68 +1,80 @@ import {ApolloClient, useApolloClient} from '@apollo/client'; +import uniq from 'lodash/uniq'; import React from 'react'; -import {assertUnreachable} from '../app/Util'; -import {LiveDataForNode, buildLiveDataForNode} from '../asset-graph/Utils'; +import {observeAssetEventsInRuns} from '../asset-graph/AssetRunLogObserver'; +import {LiveDataForNode, buildLiveDataForNode, tokenForAssetKey} from '../asset-graph/Utils'; import { AssetGraphLiveQuery, AssetGraphLiveQueryVariables, - AssetLatestInfoFragment, AssetNodeLiveFragment, } from '../asset-graph/types/useLiveDataForAssetKeys.types'; -import { - ASSETS_GRAPH_LIVE_QUERY, - ASSET_LATEST_INFO_FRAGMENT, - ASSET_NODE_LIVE_FRAGMENT, -} from '../asset-graph/useLiveDataForAssetKeys'; +import {ASSETS_GRAPH_LIVE_QUERY} from '../asset-graph/useLiveDataForAssetKeys'; import {AssetKeyInput} from '../graphql/types'; import {isDocumentVisible, useDocumentVisibility} from '../hooks/useDocumentVisibility'; +import {useDidLaunchEvent} from '../runs/RunUtils'; const _assetKeyListeners: Record> = {}; +let providerListener = (_key: string, _data?: LiveDataForNode) => {}; +const _cache: Record = {}; + +export function useAssetLiveData(assetKey: AssetKeyInput) { + const {liveDataByNode} = useAssetsLiveData(React.useMemo(() => [assetKey], [assetKey])); + return liveDataByNode[JSON.stringify(assetKey.path)]; +} -/* - * Note: This hook may return partial data since it will fetch assets in chunks/batches. - */ export function useAssetsLiveData(assetKeys: AssetKeyInput[]) { - const [data, setData] = React.useState>({}); + const [data, setData] = React.useState>({}); + const [isRefreshing, setIsRefreshing] = React.useState(false); - const client = useApolloClient(); - const setNeedsImmediateFetch = React.useContext(AssetLiveDataContext).setNeedsImmediateFetch; + const {setNeedsImmediateFetch, onSubscribed, onUnsubscribed} = + React.useContext(AssetLiveDataContext); React.useEffect(() => { - const setDataSingle = (stringKey: string, assetData: LiveDataForNode) => { + const setDataSingle = (stringKey: string, assetData?: LiveDataForNode) => { setData((data) => { - return {...data, [stringKey]: assetData}; + const copy = {...data}; + if (!assetData) { + delete copy[stringKey]; + } else { + copy[stringKey] = assetData; + } + return copy; }); }; assetKeys.forEach((key) => { - _subscribeToAssetKey(client, key, setDataSingle, setNeedsImmediateFetch); + _subscribeToAssetKey(key, setDataSingle, setNeedsImmediateFetch); }); + onSubscribed(); return () => { assetKeys.forEach((key) => { _unsubscribeToAssetKey(key, setDataSingle); }); + onUnsubscribed(); }; - }, [assetKeys, client, setNeedsImmediateFetch]); - - return data; -} + }, [assetKeys, onSubscribed, onUnsubscribed, setNeedsImmediateFetch]); -function _getAssetFromCache(client: ApolloClient, uniqueId: string) { - const cachedAssetData = client.readFragment({ - fragment: ASSET_NODE_LIVE_FRAGMENT, - fragmentName: 'AssetNodeLiveFragment', - id: `assetNodeLiveFragment-${uniqueId}`, - }); - const cachedLatestInfo = client.readFragment({ - fragment: ASSET_LATEST_INFO_FRAGMENT, - fragmentName: 'AssetLatestInfoFragment', - id: `assetLatestInfoFragment-${uniqueId}`, - }); - if (cachedAssetData && cachedLatestInfo) { - return {cachedAssetData, cachedLatestInfo}; - } else { - return null; - } + return { + liveDataByNode: data, + refresh: React.useCallback(() => { + _resetLastFetchedOrRequested(assetKeys); + setNeedsImmediateFetch(); + setIsRefreshing(true); + }, [setNeedsImmediateFetch, assetKeys]), + refreshing: React.useMemo(() => { + for (const key of assetKeys) { + const stringKey = JSON.stringify(key.path); + if (!lastFetchedOrRequested[stringKey]?.fetched) { + return true; + } + } + setTimeout(() => { + setIsRefreshing(false); + }); + return false; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [assetKeys, data, isRefreshing]), + }; } async function _queryAssetKeys(client: ApolloClient, assetKeys: AssetKeyInput[]) { @@ -73,24 +85,18 @@ async function _queryAssetKeys(client: ApolloClient, assetKeys: AssetKeyInp assetKeys, }, }); + const nodesByKey: Record = {}; + const liveDataByKey: Record = {}; data.assetNodes.forEach((assetNode) => { - _assetKeyToUniqueId[JSON.stringify(assetNode.assetKey.path)] = assetNode.id; - client.writeFragment({ - fragment: ASSET_NODE_LIVE_FRAGMENT, - fragmentName: 'AssetNodeLiveFragment', - id: `assetNodeLiveFragment-${assetNode.id}`, - data: assetNode, - }); + const id = JSON.stringify(assetNode.assetKey.path); + nodesByKey[id] = assetNode; }); data.assetsLatestInfo.forEach((assetLatestInfo) => { - _assetKeyToUniqueId[JSON.stringify(assetLatestInfo.assetKey.path)] = assetLatestInfo.id; - client.writeFragment({ - fragment: ASSET_LATEST_INFO_FRAGMENT, - fragmentName: 'AssetLatestInfoFragment', - id: `assetLatestInfoFragment-${assetLatestInfo.id}`, - data: assetLatestInfo, - }); + const id = JSON.stringify(assetLatestInfo.assetKey.path); + liveDataByKey[id] = buildLiveDataForNode(nodesByKey[id]!, assetLatestInfo); }); + Object.assign(_cache, liveDataByKey); + return liveDataByKey; } // How many assets to fetch at once @@ -100,14 +106,18 @@ export const BATCH_SIZE = 50; const BATCHING_INTERVAL = 250; export const SUBSCRIPTION_IDLE_POLL_RATE = 30 * 1000; -// const SUBSCRIPTION_MAX_POLL_RATE = 2 * 1000; +const SUBSCRIPTION_MAX_POLL_RATE = 2 * 1000; -type DataForNodeListener = (stringKey: string, data: LiveDataForNode) => void; +type DataForNodeListener = (stringKey: string, data?: LiveDataForNode) => void; const AssetLiveDataContext = React.createContext<{ setNeedsImmediateFetch: () => void; + onSubscribed: () => void; + onUnsubscribed: () => void; }>({ setNeedsImmediateFetch: () => {}, + onSubscribed: () => {}, + onUnsubscribed: () => {}, }); // Map of asset keys to their last fetched time and last requested time @@ -116,16 +126,18 @@ const lastFetchedOrRequested: Record< {fetched: number; requested?: undefined} | {requested: number; fetched?: undefined} | null > = {}; -export const _testOnly_resetLastFetchedOrRequested = () => { - if (typeof jest !== 'undefined') { - Object.keys(lastFetchedOrRequested).forEach((key) => { +export const _resetLastFetchedOrRequested = (keys?: AssetKeyInput[]) => { + (keys?.map((key) => JSON.stringify(key.path)) ?? Object.keys(lastFetchedOrRequested)).forEach( + (key) => { delete lastFetchedOrRequested[key]; - }); - } + }, + ); }; export const AssetLiveDataProvider = ({children}: {children: React.ReactNode}) => { const [needsImmediateFetch, setNeedsImmediateFetch] = React.useState(false); + const [allObservedKeys, setAllObservedKeys] = React.useState([]); + const [cache, setCache] = React.useState>({}); const client = useApolloClient(); @@ -159,6 +171,58 @@ export const AssetLiveDataProvider = ({children}: {children: React.ReactNode}) = }; }, [client, needsImmediateFetch]); + React.useEffect(() => { + providerListener = (stringKey, assetData) => { + setCache((data) => { + const copy = {...data}; + if (!assetData) { + delete copy[stringKey]; + } else { + copy[stringKey] = assetData; + } + return copy; + }); + }; + }, []); + + useDidLaunchEvent(() => { + _resetLastFetchedOrRequested(); + setNeedsImmediateFetch(true); + }, SUBSCRIPTION_MAX_POLL_RATE); + + React.useEffect(() => { + const assetKeyTokens = new Set(allObservedKeys.map(tokenForAssetKey)); + const dataForObservedKeys = allObservedKeys + .map((key) => cache[tokenForAssetKey(key)]) + .filter((n) => n) as LiveDataForNode[]; + + const assetStepKeys = new Set(dataForObservedKeys.flatMap((n) => n.opNames)); + + const runInProgressId = uniq( + dataForObservedKeys.flatMap((p) => [ + ...p.unstartedRunIds, + ...p.inProgressRunIds, + ...p.assetChecks + .map((c) => c.executionForLatestMaterialization) + .filter(Boolean) + .map((e) => e!.runId), + ]), + ).sort(); + + const unobserve = observeAssetEventsInRuns(runInProgressId, (events) => { + if ( + events.some( + (e) => + (e.assetKey && assetKeyTokens.has(tokenForAssetKey(e.assetKey))) || + (e.stepKey && assetStepKeys.has(e.stepKey)), + ) + ) { + _resetLastFetchedOrRequested(); + } + }); + return unobserve; + }, [allObservedKeys, cache]); + return ( { setNeedsImmediateFetch(true); }, + onSubscribed: () => { + setAllObservedKeys(getAllAssetKeysWithListeners()); + }, + onUnsubscribed: () => { + setAllObservedKeys(getAllAssetKeysWithListeners()); + }, }), [], )} @@ -193,27 +263,7 @@ async function _batchedQueryAssets( requested: requestTime, }; }); - await _queryAssetKeys(client, assetKeys); - const data: Record = {}; - assetKeys.forEach((key) => { - const stringKey = JSON.stringify(key.path); - const uniqueId = _assetKeyToUniqueId[stringKey]; - if (!uniqueId) { - assertUnreachable( - `Expected uniqueID for assetKey to be known after fetching it's data: ${stringKey}` as never, - ); - return; - } - const cachedData = _getAssetFromCache(client, uniqueId)!; - if (cachedData) { - const {cachedAssetData, cachedLatestInfo} = cachedData; - data[stringKey] = buildLiveDataForNode(cachedAssetData, cachedLatestInfo); - } else { - assertUnreachable( - ('Apollo failed to populate cache for asset key: ' + JSON.stringify(key.path)) as never, - ); - } - }); + const data = await _queryAssetKeys(client, assetKeys); const fetchedTime = Date.now(); assetKeys.forEach((key) => { lastFetchedOrRequested[JSON.stringify(key.path)] = { @@ -229,7 +279,6 @@ async function _batchedQueryAssets( } function _subscribeToAssetKey( - client: ApolloClient, assetKey: AssetKeyInput, setData: DataForNodeListener, setNeedsImmediateFetch: () => void, @@ -237,11 +286,9 @@ function _subscribeToAssetKey( const stringKey = JSON.stringify(assetKey.path); _assetKeyListeners[stringKey] = _assetKeyListeners[stringKey] || []; _assetKeyListeners[stringKey]!.push(setData); - const uniqueId = _assetKeyToUniqueId[stringKey]; - const cachedData = uniqueId ? _getAssetFromCache(client, uniqueId) : null; + const cachedData = _cache[stringKey]; if (cachedData) { - const {cachedAssetData, cachedLatestInfo} = cachedData; - setData(stringKey, buildLiveDataForNode(cachedAssetData, cachedLatestInfo)); + setData(stringKey, cachedData); } else { setNeedsImmediateFetch(); } @@ -290,6 +337,7 @@ function fetchData(client: ApolloClient) { _batchedQueryAssets(_determineAssetsToFetch(), client, (data) => { Object.entries(data).forEach(([key, assetData]) => { const listeners = _assetKeyListeners[key]; + providerListener(key, assetData); if (!listeners) { return; } @@ -300,5 +348,25 @@ function fetchData(client: ApolloClient) { }); } -// Map of AssetKeyInput to its unique asset ID. We won't know until we query because the backend implementation depends on the repository location and name. -const _assetKeyToUniqueId: Record = {}; +function getAllAssetKeysWithListeners(): AssetKeyInput[] { + return Object.keys(_assetKeyListeners).map((key) => ({path: JSON.parse(key)})); +} + +export function _setCacheEntryForTest(assetKey: AssetKeyInput, data?: LiveDataForNode) { + if (process.env.STORYBOOK || typeof jest !== 'undefined') { + const stringKey = JSON.stringify(assetKey.path); + if (data) { + _cache[stringKey] = data; + } else { + delete _cache[stringKey]; + } + const listeners = _assetKeyListeners[stringKey]; + providerListener(stringKey, data); + if (!listeners) { + return; + } + listeners.forEach((listener) => { + listener(stringKey, data); + }); + } +} diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-data/__tests__/AssetLiveDataProvider.test.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-data/__tests__/AssetLiveDataProvider.test.tsx index a8007b4d5f9d9..694bff916f4d2 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-data/__tests__/AssetLiveDataProvider.test.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-data/__tests__/AssetLiveDataProvider.test.tsx @@ -4,49 +4,22 @@ import {MockedProvider, MockedResponse} from '@apollo/client/testing'; import {render, act, waitFor} from '@testing-library/react'; import React from 'react'; -import { - AssetGraphLiveQuery, - AssetGraphLiveQueryVariables, -} from '../../asset-graph/types/useLiveDataForAssetKeys.types'; -import {ASSETS_GRAPH_LIVE_QUERY} from '../../asset-graph/useLiveDataForAssetKeys'; -import { - AssetKey, - AssetKeyInput, - buildAssetKey, - buildAssetLatestInfo, - buildAssetNode, -} from '../../graphql/types'; -import {buildQueryMock, getMockResultFn} from '../../testing/mocking'; +import {AssetKey, AssetKeyInput, buildAssetKey} from '../../graphql/types'; +import {getMockResultFn} from '../../testing/mocking'; import { AssetLiveDataProvider, SUBSCRIPTION_IDLE_POLL_RATE, - _testOnly_resetLastFetchedOrRequested, + _resetLastFetchedOrRequested, useAssetsLiveData, } from '../AssetLiveDataProvider'; +import {buildMockedAssetGraphLiveQuery} from './util'; + Object.defineProperty(document, 'visibilityState', {value: 'visible', writable: true}); Object.defineProperty(document, 'hidden', {value: false, writable: true}); -function buildMockedQuery(assetKeys: AssetKeyInput[]) { - return buildQueryMock({ - query: ASSETS_GRAPH_LIVE_QUERY, - variables: { - // strip __typename - assetKeys: assetKeys.map((assetKey) => ({path: assetKey.path})), - }, - data: { - assetNodes: assetKeys.map((assetKey) => - buildAssetNode({assetKey: buildAssetKey(assetKey), id: JSON.stringify(assetKey)}), - ), - assetsLatestInfo: assetKeys.map((assetKey) => - buildAssetLatestInfo({assetKey: buildAssetKey(assetKey), id: JSON.stringify(assetKey)}), - ), - }, - }); -} - afterEach(() => { - _testOnly_resetLastFetchedOrRequested(); + _resetLastFetchedOrRequested(); }); function Test({ @@ -56,7 +29,7 @@ function Test({ mocks: MockedResponse[]; hooks: { keys: AssetKeyInput[]; - hookResult: (data: ReturnType) => void; + hookResult: (data: ReturnType['liveDataByNode']) => void; }[]; }) { function Hook({ @@ -64,9 +37,9 @@ function Test({ hookResult, }: { keys: AssetKeyInput[]; - hookResult: (data: ReturnType) => void; + hookResult: (data: ReturnType['liveDataByNode']) => void; }) { - hookResult(useAssetsLiveData(keys)); + hookResult(useAssetsLiveData(keys).liveDataByNode); return
; } return ( @@ -83,8 +56,8 @@ function Test({ describe('AssetLiveDataProvider', () => { it('provides asset data and uses cache if recently fetched', async () => { const assetKeys = [buildAssetKey({path: ['key1']})]; - const mockedQuery = buildMockedQuery(assetKeys); - const mockedQuery2 = buildMockedQuery(assetKeys); + const mockedQuery = buildMockedAssetGraphLiveQuery(assetKeys); + const mockedQuery2 = buildMockedAssetGraphLiveQuery(assetKeys); const resultFn = getMockResultFn(mockedQuery); const resultFn2 = getMockResultFn(mockedQuery2); @@ -98,13 +71,16 @@ describe('AssetLiveDataProvider', () => { // Initially an empty object expect(resultFn).toHaveBeenCalledTimes(0); - expect(hookResult.mock.results[0]!.value).toEqual(undefined); + expect(hookResult.mock.calls[0]!.value).toEqual(undefined); act(() => { jest.runOnlyPendingTimers(); }); expect(resultFn).toHaveBeenCalled(); + await waitFor(() => { + expect(hookResult.mock.calls[1]!.value).not.toEqual({}); + }); expect(resultFn2).not.toHaveBeenCalled(); // Re-render with the same asset keys and expect the cache to be used this time. @@ -118,14 +94,14 @@ describe('AssetLiveDataProvider', () => { // Initially an empty object expect(resultFn2).not.toHaveBeenCalled(); - expect(hookResult2.mock.results[0]!.value).toEqual(undefined); + expect(hookResult2.mock.calls[0][0]).toEqual({}); act(() => { jest.runOnlyPendingTimers(); }); // Not called because we use the cache instead expect(resultFn2).not.toHaveBeenCalled(); - expect(hookResult2.mock.results[1]).toEqual(hookResult.mock.results[1]); + expect(hookResult2.mock.calls[1]).toEqual(hookResult.mock.calls[1]); await act(async () => { await Promise.resolve(); @@ -136,13 +112,13 @@ describe('AssetLiveDataProvider', () => { jest.advanceTimersByTime(SUBSCRIPTION_IDLE_POLL_RATE + 1); }); expect(resultFn2).toHaveBeenCalled(); - expect(hookResult2.mock.results[2]).toEqual(hookResult.mock.results[1]); + expect(hookResult2.mock.calls[1]).toEqual(hookResult.mock.calls[1]); }); it('obeys document visibility', async () => { const assetKeys = [buildAssetKey({path: ['key1']})]; - const mockedQuery = buildMockedQuery(assetKeys); - const mockedQuery2 = buildMockedQuery(assetKeys); + const mockedQuery = buildMockedAssetGraphLiveQuery(assetKeys); + const mockedQuery2 = buildMockedAssetGraphLiveQuery(assetKeys); const resultFn = getMockResultFn(mockedQuery); const resultFn2 = getMockResultFn(mockedQuery2); @@ -156,7 +132,7 @@ describe('AssetLiveDataProvider', () => { // Initially an empty object expect(resultFn).toHaveBeenCalledTimes(0); - expect(hookResult.mock.results[0]!.value).toEqual(undefined); + expect(hookResult.mock.calls[0]!.value).toEqual(undefined); act(() => { jest.runOnlyPendingTimers(); @@ -176,14 +152,14 @@ describe('AssetLiveDataProvider', () => { // Initially an empty object expect(resultFn2).not.toHaveBeenCalled(); - expect(hookResult2.mock.results[0]!.value).toEqual(undefined); + expect(hookResult2.mock.calls[0]!.value).toEqual(undefined); act(() => { jest.runOnlyPendingTimers(); }); // Not called because we use the cache instead expect(resultFn2).not.toHaveBeenCalled(); - expect(hookResult2.mock.results[1]).toEqual(hookResult.mock.results[1]); + expect(hookResult2.mock.calls[1]).toEqual(hookResult.mock.calls[1]); await act(async () => { await Promise.resolve(); @@ -219,8 +195,8 @@ describe('AssetLiveDataProvider', () => { const chunk1 = assetKeys.slice(0, 50); const chunk2 = assetKeys.slice(50, 100); - const mockedQuery = buildMockedQuery(chunk1); - const mockedQuery2 = buildMockedQuery(chunk2); + const mockedQuery = buildMockedAssetGraphLiveQuery(chunk1); + const mockedQuery2 = buildMockedAssetGraphLiveQuery(chunk2); const resultFn = getMockResultFn(mockedQuery); const resultFn2 = getMockResultFn(mockedQuery2); @@ -231,7 +207,7 @@ describe('AssetLiveDataProvider', () => { // Initially an empty object expect(resultFn).not.toHaveBeenCalled(); - expect(hookResult.mock.results[0]!.value).toEqual(undefined); + expect(hookResult.mock.calls[0][0]).toEqual({}); act(() => { jest.runOnlyPendingTimers(); @@ -265,8 +241,8 @@ describe('AssetLiveDataProvider', () => { const hook2Keys = assetKeys.slice(33, 66); const hook3Keys = assetKeys.slice(66, 100); - const mockedQuery = buildMockedQuery(chunk1); - const mockedQuery2 = buildMockedQuery(chunk2); + const mockedQuery = buildMockedAssetGraphLiveQuery(chunk1); + const mockedQuery2 = buildMockedAssetGraphLiveQuery(chunk2); const resultFn = getMockResultFn(mockedQuery); const resultFn2 = getMockResultFn(mockedQuery2); @@ -324,9 +300,9 @@ describe('AssetLiveDataProvider', () => { secondPrioritizedFetchKeys.push(...fetch1Keys); - const mockedQuery = buildMockedQuery(fetch1Keys); - const mockedQuery2 = buildMockedQuery(firstPrioritizedFetchKeys); - const mockedQuery3 = buildMockedQuery(secondPrioritizedFetchKeys); + const mockedQuery = buildMockedAssetGraphLiveQuery(fetch1Keys); + const mockedQuery2 = buildMockedAssetGraphLiveQuery(firstPrioritizedFetchKeys); + const mockedQuery3 = buildMockedAssetGraphLiveQuery(secondPrioritizedFetchKeys); const resultFn = getMockResultFn(mockedQuery); const resultFn2 = getMockResultFn(mockedQuery2); @@ -343,6 +319,9 @@ describe('AssetLiveDataProvider', () => { await waitFor(() => { expect(resultFn).toHaveBeenCalled(); }); + await waitFor(() => { + expect(hookResult.mock.calls[1]!.value).not.toEqual({}); + }); expect(resultFn2).not.toHaveBeenCalled(); expect(resultFn3).not.toHaveBeenCalled(); diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-data/__tests__/util.ts b/js_modules/dagster-ui/packages/ui-core/src/asset-data/__tests__/util.ts new file mode 100644 index 0000000000000..754ff1d12c51f --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-data/__tests__/util.ts @@ -0,0 +1,30 @@ +import { + AssetGraphLiveQuery, + AssetGraphLiveQueryVariables, +} from '../../asset-graph/types/useLiveDataForAssetKeys.types'; +import {ASSETS_GRAPH_LIVE_QUERY} from '../../asset-graph/useLiveDataForAssetKeys'; +import { + AssetKeyInput, + buildAssetNode, + buildAssetKey, + buildAssetLatestInfo, +} from '../../graphql/types'; +import {buildQueryMock} from '../../testing/mocking'; + +export function buildMockedAssetGraphLiveQuery(assetKeys: AssetKeyInput[]) { + return buildQueryMock({ + query: ASSETS_GRAPH_LIVE_QUERY, + variables: { + // strip __typename + assetKeys: assetKeys.map((assetKey) => ({path: assetKey.path})), + }, + data: { + assetNodes: assetKeys.map((assetKey) => + buildAssetNode({assetKey: buildAssetKey(assetKey), id: JSON.stringify(assetKey)}), + ), + assetsLatestInfo: assetKeys.map((assetKey) => + buildAssetLatestInfo({assetKey: buildAssetKey(assetKey), id: JSON.stringify(assetKey)}), + ), + }, + }); +} diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetEdges.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetEdges.tsx index e8ccb32b21213..92a997c1a002f 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetEdges.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetEdges.tsx @@ -9,15 +9,31 @@ export const AssetEdges: React.FC<{ highlighted: string | null; strokeWidth?: number; baseColor?: string; -}> = ({edges, highlighted, strokeWidth = 4, baseColor = Colors.KeylineGray}) => { + viewportRect: {top: number; left: number; right: number; bottom: number}; +}> = ({edges, highlighted, strokeWidth = 4, baseColor = Colors.KeylineGray, viewportRect}) => { // Note: we render the highlighted edges twice, but it's so that the first item with // all the edges in it can remain memoized. + + const intersectedEdges = edges.filter((edge) => doesViewportContainEdge(edge, viewportRect)); + const visibleToFromEdges = intersectedEdges.filter( + (edge) => + doesViewportContainPoint(edge.from, viewportRect) || + doesViewportContainPoint(edge.to, viewportRect), + ); return ( - 50 ? visibleToFromEdges : intersectedEdges} + strokeWidth={strokeWidth} + viewportRect={viewportRect} + /> + highlighted === fromId || highlighted === toId)} + edges={intersectedEdges.filter( + ({fromId, toId}) => highlighted === fromId || highlighted === toId, + )} strokeWidth={strokeWidth} /> @@ -28,6 +44,7 @@ const AssetEdgeSet: React.FC<{ edges: AssetLayoutEdge[]; color: string; strokeWidth: number; + viewportRect: {top: number; left: number; right: number; bottom: number}; }> = React.memo(({edges, color, strokeWidth}) => ( <> @@ -55,3 +72,40 @@ const AssetEdgeSet: React.FC<{ ))} )); + +//https://stackoverflow.com/a/20925869/1162881 +function doesViewportContainEdge( + edge: {from: {x: number; y: number}; to: {x: number; y: number}}, + viewportRect: {top: number; left: number; right: number; bottom: number}, +) { + return ( + isOverlapping1D( + Math.max(edge.from.x, edge.to.x), + Math.max(viewportRect.left, viewportRect.right), + Math.min(edge.from.x, edge.to.x), + Math.min(viewportRect.left, viewportRect.right), + ) && + isOverlapping1D( + Math.max(edge.from.y, edge.to.y), + Math.max(viewportRect.top, viewportRect.bottom), + Math.min(edge.from.y, edge.to.y), + Math.min(viewportRect.top, viewportRect.bottom), + ) + ); +} + +function isOverlapping1D(xmax1: number, xmax2: number, xmin1: number, xmin2: number) { + return xmax1 >= xmin2 && xmax2 >= xmin1; +} + +function doesViewportContainPoint( + point: {x: number; y: number}, + viewportRect: {top: number; left: number; right: number; bottom: number}, +) { + return ( + point.x >= viewportRect.left && + point.x <= viewportRect.right && + point.y >= viewportRect.top && + point.y <= viewportRect.bottom + ); +} diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx index fb9c8c1f195df..a673b0c79c285 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx @@ -17,13 +17,12 @@ import {useHistory} from 'react-router-dom'; import styled from 'styled-components'; import {useFeatureFlags} from '../app/Flags'; -import {QueryRefreshCountdown, QueryRefreshState} from '../app/QueryRefresh'; import {LaunchAssetExecutionButton} from '../assets/LaunchAssetExecutionButton'; import {LaunchAssetObservationButton} from '../assets/LaunchAssetObservationButton'; import {AssetKey} from '../assets/types'; import {DEFAULT_MAX_ZOOM, SVGViewport} from '../graph/SVGViewport'; import {useAssetLayout} from '../graph/asyncGraphLayout'; -import {closestNodeInDirection} from '../graph/common'; +import {closestNodeInDirection, isNodeOffscreen} from '../graph/common'; import { GraphExplorerOptions, OptionsOverlay, @@ -48,12 +47,11 @@ import {AssetGroupNode} from './AssetGroupNode'; import {AssetNode, AssetNodeMinimal} from './AssetNode'; import {AssetNodeLink} from './ForeignNode'; import {SidebarAssetInfo} from './SidebarAssetInfo'; -import {GraphData, graphHasCycles, LiveData, GraphNode, tokenForAssetKey} from './Utils'; +import {GraphData, graphHasCycles, GraphNode, tokenForAssetKey} from './Utils'; import {AssetGraphLayout} from './layout'; import {AssetNodeForGraphQueryFragment} from './types/useAssetGraphData.types'; import {AssetGraphFetchScope, AssetGraphQueryItem, useAssetGraphData} from './useAssetGraphData'; import {AssetLocation, useFindAssetLocation} from './useFindAssetLocation'; -import {useLiveDataForAssetKeys} from './useLiveDataForAssetKeys'; type AssetNode = AssetNodeForGraphQueryFragment; @@ -78,13 +76,10 @@ export const AssetGraphExplorer: React.FC = (props) => { assetGraphData, fullAssetGraphData, graphQueryItems, - graphAssetKeys, allAssetKeys, applyingEmptyDefault, } = useAssetGraphData(props.explorerPath.opsQuery, props.fetchOptions); - const {liveDataByNode, liveDataRefreshState} = useLiveDataForAssetKeys(graphAssetKeys); - return ( {() => { @@ -111,8 +106,6 @@ export const AssetGraphExplorer: React.FC = (props) => { allAssetKeys={allAssetKeys} graphQueryItems={graphQueryItems} applyingEmptyDefault={applyingEmptyDefault} - liveDataRefreshState={liveDataRefreshState} - liveDataByNode={liveDataByNode} {...props} /> ); @@ -126,8 +119,6 @@ type WithDataProps = { assetGraphData: GraphData; fullAssetGraphData: GraphData; graphQueryItems: AssetGraphQueryItem[]; - liveDataByNode: LiveData; - liveDataRefreshState: QueryRefreshState; applyingEmptyDefault: boolean; } & Props; @@ -137,8 +128,6 @@ const AssetGraphExplorerWithData: React.FC = ({ explorerPath, onChangeExplorerPath, onNavigateToSourceAssetNode: onNavigateToSourceAssetNode, - liveDataRefreshState, - liveDataByNode, assetGraphData, fullAssetGraphData, graphQueryItems, @@ -338,9 +327,10 @@ const AssetGraphExplorerWithData: React.FC = ({ maxZoom={DEFAULT_MAX_ZOOM} maxAutocenterZoom={1.0} > - {({scale}) => ( + {({scale}, viewportRect) => ( = ({ /> {Object.values(layout.groups) + .filter((node) => !isNodeOffscreen(node.bounds, viewportRect)) .sort((a, b) => a.id.length - b.id.length) .map((group) => ( = ({ ))} - {Object.values(layout.nodes).map(({id, bounds}) => { - const graphNode = assetGraphData.nodes[id]!; - const path = JSON.parse(id); - if (allowGroupsOnlyZoomLevel && scale < GROUPS_ONLY_SCALE) { - return; - } - return ( - setHighlighted(id)} - onMouseLeave={() => setHighlighted(null)} - onClick={(e) => onSelectNode(e, {path}, graphNode)} - onDoubleClick={(e) => { - viewportEl.current?.zoomToSVGBox(bounds, true, 1.2); - e.stopPropagation(); - }} - style={{overflow: 'visible'}} - > - {!graphNode ? ( - - ) : scale < MINIMAL_SCALE ? ( - - ) : ( - - )} - - ); - })} + {Object.values(layout.nodes) + .filter((node) => !isNodeOffscreen(node.bounds, viewportRect)) + .map(({id, bounds}) => { + const graphNode = assetGraphData.nodes[id]!; + const path = JSON.parse(id); + if (allowGroupsOnlyZoomLevel && scale < GROUPS_ONLY_SCALE) { + return; + } + return ( + setHighlighted(id)} + onMouseLeave={() => setHighlighted(null)} + onClick={(e) => onSelectNode(e, {path}, graphNode)} + onDoubleClick={(e) => { + viewportEl.current?.zoomToSVGBox(bounds, true, 1.2); + e.stopPropagation(); + }} + style={{overflow: 'visible'}} + > + {!graphNode ? ( + + ) : scale < MINIMAL_SCALE ? ( + + ) : ( + + )} + + ); + })} )} @@ -436,10 +427,6 @@ const AssetGraphExplorerWithData: React.FC = ({ - = ({ /> = ({ - + diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorerSidebar.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorerSidebar.tsx index ab92b335d0c1d..8e85cd878c94c 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorerSidebar.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorerSidebar.tsx @@ -486,7 +486,7 @@ const Node = ({ }} selectNode={selectNode} /> - + {!isAssetNode || @@ -560,7 +560,7 @@ const Node = ({ }} /> {upstream.length || downstream.length ? : null} - {upstream.length > 1 ? ( + {upstream.length ? ( 0 ? {side: 'left', width: 1, color: Colors.KeylineGray} : undefined} + border={{side: 'left', width: 1, color: Colors.KeylineGray}} + style={{position: 'relative'}} > {sofar} diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetNode.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetNode.tsx index 9f1561cef6632..17137ae2f2f0b 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetNode.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetNode.tsx @@ -7,6 +7,7 @@ import {Link} from 'react-router-dom'; import styled, {CSSObject} from 'styled-components'; import {withMiddleTruncation} from '../app/Util'; +import {useAssetLiveData} from '../asset-data/AssetLiveDataProvider'; import {PartitionCountTags} from '../assets/AssetNodePartitionCounts'; import {StaleReasonsTags} from '../assets/Stale'; import {assetDetailsPathForKey} from '../assets/assetDetailsPathForKey'; @@ -22,12 +23,12 @@ import {AssetNodeFragment} from './types/AssetNode.types'; export const AssetNode: React.FC<{ definition: AssetNodeFragment; - liveData?: LiveDataForNode; selected: boolean; -}> = React.memo(({definition, selected, liveData}) => { +}> = React.memo(({definition, selected}) => { const displayName = definition.assetKey.path[definition.assetKey.path.length - 1]!; const isSource = definition.isSource; + const liveData = useAssetLiveData(definition.assetKey); return ( @@ -206,10 +207,10 @@ const AssetNodeChecksRow: React.FC<{ export const AssetNodeMinimal: React.FC<{ selected: boolean; - liveData?: LiveDataForNode; definition: AssetNodeFragment; -}> = ({selected, definition, liveData}) => { +}> = ({selected, definition}) => { const {isSource, assetKey} = definition; + const liveData = useAssetLiveData(assetKey); const {border, background} = buildAssetNodeStatusContent({assetKey, definition, liveData}); const displayName = assetKey.path[assetKey.path.length - 1]!; diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/SidebarAssetInfo.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/SidebarAssetInfo.tsx index ada34c3656843..498655b9240e6 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/SidebarAssetInfo.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/SidebarAssetInfo.tsx @@ -5,6 +5,7 @@ import {Link} from 'react-router-dom'; import styled from 'styled-components'; import {COMMON_COLLATOR} from '../app/Util'; +import {useAssetLiveData} from '../asset-data/AssetLiveDataProvider'; import {ASSET_NODE_CONFIG_FRAGMENT} from '../assets/AssetConfig'; import {AssetDefinedInMultipleReposNotice} from '../assets/AssetDefinedInMultipleReposNotice'; import { @@ -34,21 +35,15 @@ import {buildRepoAddress} from '../workspace/buildRepoAddress'; import {RepoAddress} from '../workspace/types'; import {workspacePathFromAddress} from '../workspace/workspacePath'; -import { - LiveDataForNode, - displayNameForAssetKey, - GraphNode, - nodeDependsOnSelf, - stepKeyForAsset, -} from './Utils'; +import {displayNameForAssetKey, GraphNode, nodeDependsOnSelf, stepKeyForAsset} from './Utils'; import {SidebarAssetQuery, SidebarAssetQueryVariables} from './types/SidebarAssetInfo.types'; import {AssetNodeForGraphQueryFragment} from './types/useAssetGraphData.types'; export const SidebarAssetInfo: React.FC<{ graphNode: GraphNode; - liveData?: LiveDataForNode; -}> = ({graphNode, liveData}) => { +}> = ({graphNode}) => { const {assetKey, definition} = graphNode; + const liveData = useAssetLiveData(assetKey); const partitionHealthRefreshHint = healthRefreshHintFromLiveData(liveData); const partitionHealthData = usePartitionHealthData( [assetKey], diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/Utils.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/Utils.tsx index d0382418b5a89..37bce3c8933bb 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/Utils.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/Utils.tsx @@ -148,6 +148,7 @@ export interface LiveDataForNode { numPartitions: number; numFailed: number; } | null; + opNames: string[]; } export const MISSING_LIVE_DATA: LiveDataForNode = { @@ -162,6 +163,7 @@ export const MISSING_LIVE_DATA: LiveDataForNode = { staleStatus: null, staleCauses: [], assetChecks: [], + opNames: [], stepKey: '', }; @@ -217,6 +219,7 @@ export const buildLiveDataForNode = ( unstartedRunIds: assetLatestInfo?.unstartedRunIds || [], partitionStats: assetNode.partitionStats || null, runWhichFailedToMaterialize, + opNames: assetNode.opNames, }; }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/__fixtures__/AssetNode.fixtures.ts b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/__fixtures__/AssetNode.fixtures.ts index 60eaa202e0e3a..f68d6a1f2bac8 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/__fixtures__/AssetNode.fixtures.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/__fixtures__/AssetNode.fixtures.ts @@ -8,6 +8,7 @@ import { buildAssetCheckExecution, buildAssetCheckEvaluation, buildAssetCheck, + buildAssetKey, } from '../../graphql/types'; import {LiveDataForNode} from '../Utils'; import {AssetNodeFragment} from '../types/AssetNode.types'; @@ -75,14 +76,14 @@ export const AssetNodeFragmentSource: AssetNodeFragment = { export const AssetNodeFragmentPartitioned: AssetNodeFragment = { ...AssetNodeFragmentBasic, - assetKey: {__typename: 'AssetKey', path: ['asset1']}, + assetKey: {__typename: 'AssetKey', path: ['asset_partioned']}, description: 'This is a partitioned asset description', - id: '["asset1"]', + id: '["asset_partioned"]', isPartitioned: true, }; export const LiveDataForNodeRunStartedNotMaterializing: LiveDataForNode = { - stepKey: 'asset1', + stepKey: 'asset2', unstartedRunIds: ['ABCDEF'], inProgressRunIds: [], lastMaterialization: null, @@ -94,10 +95,11 @@ export const LiveDataForNodeRunStartedNotMaterializing: LiveDataForNode = { assetChecks: [], freshnessInfo: null, partitionStats: null, + opNames: [], }; export const LiveDataForNodeRunStartedMaterializing: LiveDataForNode = { - stepKey: 'asset1', + stepKey: 'asset3', unstartedRunIds: [], inProgressRunIds: ['ABCDEF'], lastMaterialization: null, @@ -109,10 +111,11 @@ export const LiveDataForNodeRunStartedMaterializing: LiveDataForNode = { assetChecks: [], freshnessInfo: null, partitionStats: null, + opNames: [], }; export const LiveDataForNodeRunFailed: LiveDataForNode = { - stepKey: 'asset1', + stepKey: 'asset4', unstartedRunIds: [], inProgressRunIds: [], lastMaterialization: null, @@ -129,10 +132,11 @@ export const LiveDataForNodeRunFailed: LiveDataForNode = { assetChecks: [], freshnessInfo: null, partitionStats: null, + opNames: [], }; export const LiveDataForNodeNeverMaterialized: LiveDataForNode = { - stepKey: 'asset1', + stepKey: 'asset5', unstartedRunIds: [], inProgressRunIds: [], lastMaterialization: null, @@ -144,10 +148,11 @@ export const LiveDataForNodeNeverMaterialized: LiveDataForNode = { assetChecks: [], freshnessInfo: null, partitionStats: null, + opNames: [], }; export const LiveDataForNodeMaterialized: LiveDataForNode = { - stepKey: 'asset1', + stepKey: 'asset6', unstartedRunIds: [], inProgressRunIds: [], lastMaterialization: { @@ -163,10 +168,11 @@ export const LiveDataForNodeMaterialized: LiveDataForNode = { assetChecks: [], freshnessInfo: null, partitionStats: null, + opNames: [], }; export const LiveDataForNodeMaterializedWithChecks: LiveDataForNode = { - stepKey: 'asset1', + stepKey: 'asset7', unstartedRunIds: [], inProgressRunIds: [], lastMaterialization: { @@ -233,6 +239,7 @@ export const LiveDataForNodeMaterializedWithChecks: LiveDataForNode = { ], freshnessInfo: null, partitionStats: null, + opNames: [], }; export const LiveDataForNodeMaterializedWithChecksOk: LiveDataForNode = { @@ -243,7 +250,7 @@ export const LiveDataForNodeMaterializedWithChecksOk: LiveDataForNode = { }; export const LiveDataForNodeMaterializedAndStale: LiveDataForNode = { - stepKey: 'asset1', + stepKey: 'asset8', unstartedRunIds: [], inProgressRunIds: [], lastMaterialization: { @@ -259,10 +266,11 @@ export const LiveDataForNodeMaterializedAndStale: LiveDataForNode = { assetChecks: [], freshnessInfo: null, partitionStats: null, + opNames: [], }; export const LiveDataForNodeMaterializedAndStaleAndOverdue: LiveDataForNode = { - stepKey: 'asset1', + stepKey: 'asset9', unstartedRunIds: [], inProgressRunIds: [], lastMaterialization: { @@ -281,10 +289,11 @@ export const LiveDataForNodeMaterializedAndStaleAndOverdue: LiveDataForNode = { currentMinutesLate: 12, }, partitionStats: null, + opNames: [], }; export const LiveDataForNodeMaterializedAndStaleAndFresh: LiveDataForNode = { - stepKey: 'asset1', + stepKey: 'asset10', unstartedRunIds: [], inProgressRunIds: [], lastMaterialization: { @@ -303,10 +312,11 @@ export const LiveDataForNodeMaterializedAndStaleAndFresh: LiveDataForNode = { currentMinutesLate: 0, }, partitionStats: null, + opNames: [], }; export const LiveDataForNodeMaterializedAndFresh: LiveDataForNode = { - stepKey: 'asset1', + stepKey: 'asset11', unstartedRunIds: [], inProgressRunIds: [], lastMaterialization: { @@ -325,10 +335,11 @@ export const LiveDataForNodeMaterializedAndFresh: LiveDataForNode = { currentMinutesLate: 0, }, partitionStats: null, + opNames: [], }; export const LiveDataForNodeMaterializedAndOverdue: LiveDataForNode = { - stepKey: 'asset1', + stepKey: 'asset12', unstartedRunIds: [], inProgressRunIds: [], lastMaterialization: { @@ -347,10 +358,11 @@ export const LiveDataForNodeMaterializedAndOverdue: LiveDataForNode = { currentMinutesLate: 12, }, partitionStats: null, + opNames: [], }; export const LiveDataForNodeFailedAndOverdue: LiveDataForNode = { - stepKey: 'asset1', + stepKey: 'asset13', unstartedRunIds: [], inProgressRunIds: [], lastMaterializationRunStatus: null, @@ -370,10 +382,11 @@ export const LiveDataForNodeFailedAndOverdue: LiveDataForNode = { currentMinutesLate: 12, }, partitionStats: null, + opNames: [], }; export const LiveDataForNodeSourceNeverObserved: LiveDataForNode = { - stepKey: 'source_asset', + stepKey: 'source_asset2', unstartedRunIds: [], inProgressRunIds: [], lastMaterialization: null, @@ -386,10 +399,11 @@ export const LiveDataForNodeSourceNeverObserved: LiveDataForNode = { freshnessInfo: null, partitionStats: null, + opNames: [], }; export const LiveDataForNodeSourceObservationRunning: LiveDataForNode = { - stepKey: 'source_asset', + stepKey: 'source_asset3', unstartedRunIds: [], inProgressRunIds: ['ABCDEF'], lastMaterialization: null, @@ -401,9 +415,10 @@ export const LiveDataForNodeSourceObservationRunning: LiveDataForNode = { assetChecks: [], freshnessInfo: null, partitionStats: null, + opNames: [], }; export const LiveDataForNodeSourceObservedUpToDate: LiveDataForNode = { - stepKey: 'source_asset', + stepKey: 'source_asset4', unstartedRunIds: [], inProgressRunIds: [], lastMaterialization: null, @@ -418,12 +433,12 @@ export const LiveDataForNodeSourceObservedUpToDate: LiveDataForNode = { staleCauses: [], assetChecks: [], freshnessInfo: null, - + opNames: [], partitionStats: null, }; export const LiveDataForNodePartitionedSomeMissing: LiveDataForNode = { - stepKey: 'partitioned_asset', + stepKey: 'partitioned_asset1', unstartedRunIds: [], inProgressRunIds: [], lastMaterialization: { @@ -444,10 +459,11 @@ export const LiveDataForNodePartitionedSomeMissing: LiveDataForNode = { numPartitions: 1500, numFailed: 0, }, + opNames: [], }; export const LiveDataForNodePartitionedSomeFailed: LiveDataForNode = { - stepKey: 'partitioned_asset', + stepKey: 'partitioned_asset2', unstartedRunIds: [], inProgressRunIds: [], lastMaterialization: { @@ -468,10 +484,11 @@ export const LiveDataForNodePartitionedSomeFailed: LiveDataForNode = { numPartitions: 1500, numFailed: 849, }, + opNames: [], }; export const LiveDataForNodePartitionedNoneMissing: LiveDataForNode = { - stepKey: 'partitioned_asset', + stepKey: 'partitioned_asset3', unstartedRunIds: [], inProgressRunIds: [], lastMaterialization: { @@ -492,10 +509,11 @@ export const LiveDataForNodePartitionedNoneMissing: LiveDataForNode = { numPartitions: 1500, numFailed: 0, }, + opNames: [], }; export const LiveDataForNodePartitionedNeverMaterialized: LiveDataForNode = { - stepKey: 'asset1', + stepKey: 'asset20', unstartedRunIds: [], inProgressRunIds: [], lastMaterialization: null, @@ -512,10 +530,11 @@ export const LiveDataForNodePartitionedNeverMaterialized: LiveDataForNode = { numPartitions: 1500, numFailed: 0, }, + opNames: [], }; export const LiveDataForNodePartitionedMaterializing: LiveDataForNode = { - stepKey: 'asset1', + stepKey: 'asset21', unstartedRunIds: ['LMAANO'], inProgressRunIds: ['ABCDEF', 'CDEFG', 'HIHKA'], lastMaterialization: null, @@ -532,10 +551,11 @@ export const LiveDataForNodePartitionedMaterializing: LiveDataForNode = { numPartitions: 1500, numFailed: 0, }, + opNames: [], }; export const LiveDataForNodePartitionedStale: LiveDataForNode = { - stepKey: 'asset1', + stepKey: 'asset22', unstartedRunIds: [], inProgressRunIds: [], lastMaterialization: { @@ -556,10 +576,11 @@ export const LiveDataForNodePartitionedStale: LiveDataForNode = { numPartitions: 1500, numFailed: 0, }, + opNames: [], }; export const LiveDataForNodePartitionedOverdue: LiveDataForNode = { - stepKey: 'asset1', + stepKey: 'asset23', unstartedRunIds: [], inProgressRunIds: [], lastMaterialization: { @@ -583,10 +604,11 @@ export const LiveDataForNodePartitionedOverdue: LiveDataForNode = { numPartitions: 1500, numFailed: 0, }, + opNames: [], }; export const LiveDataForNodePartitionedFresh: LiveDataForNode = { - stepKey: 'asset1', + stepKey: 'asset24', unstartedRunIds: [], inProgressRunIds: [], lastMaterialization: { @@ -610,10 +632,11 @@ export const LiveDataForNodePartitionedFresh: LiveDataForNode = { numPartitions: 1500, numFailed: 0, }, + opNames: [], }; export const LiveDataForNodePartitionedLatestRunFailed: LiveDataForNode = { - stepKey: 'asset1', + stepKey: 'asset25', unstartedRunIds: [], inProgressRunIds: [], lastMaterialization: null, @@ -635,6 +658,7 @@ export const LiveDataForNodePartitionedLatestRunFailed: LiveDataForNode = { numPartitions: 1500, numFailed: 1, }, + opNames: [], }; export const AssetNodeScenariosBase = [ @@ -683,14 +707,14 @@ export const AssetNodeScenariosBase = [ title: 'Materialized and Stale', liveData: LiveDataForNodeMaterializedAndStale, definition: AssetNodeFragmentBasic, - expectedText: ['Code version', 'Feb'], + expectedText: ['Upstream code version', 'Feb'], }, { title: 'Materialized and Stale and Overdue', liveData: LiveDataForNodeMaterializedAndStaleAndOverdue, definition: AssetNodeFragmentBasic, - expectedText: ['Code version', 'Overdue', 'Feb'], + expectedText: ['Upstream code version', 'Overdue', 'Feb'], }, { @@ -744,14 +768,25 @@ export const AssetNodeScenariosSource = [ { title: 'Source Asset - Not Observable', liveData: undefined, - definition: {...AssetNodeFragmentSource, isObservable: false}, + definition: { + ...AssetNodeFragmentSource, + isObservable: false, + id: '["source_asset_no"]', + assetKey: buildAssetKey({path: ['source_asset_no']}), + }, expectedText: [], }, { title: 'Source Asset - Not Observable, No Description', liveData: undefined, - definition: {...AssetNodeFragmentSource, isObservable: false, description: null}, + definition: { + ...AssetNodeFragmentSource, + isObservable: false, + description: null, + id: '["source_asset_nono"]', + assetKey: buildAssetKey({path: ['source_asset_nono']}), + }, expectedText: [], }, diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/__stories__/AssetNode.stories.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/__stories__/AssetNode.stories.tsx index a976ae1e596d7..78f0f9b2e5a16 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/__stories__/AssetNode.stories.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/__stories__/AssetNode.stories.tsx @@ -1,7 +1,9 @@ import {Box} from '@dagster-io/ui-components'; import React from 'react'; +import {_setCacheEntryForTest} from '../../asset-data/AssetLiveDataProvider'; import {KNOWN_TAGS} from '../../graph/OpTags'; +import {buildAssetKey} from '../../graphql/types'; import {AssetNode, AssetNodeMinimal} from '../AssetNode'; import {AssetNodeLink} from '../ForeignNode'; import * as Mocks from '../__fixtures__/AssetNode.fixtures'; @@ -15,7 +17,19 @@ export default { export const LiveStates = () => { const caseWithLiveData = (scenario: (typeof Mocks.AssetNodeScenariosBase)[0]) => { - const dimensions = getAssetNodeDimensions(scenario.definition); + const definitionCopy = { + ...scenario.definition, + assetKey: { + ...scenario.definition.assetKey, + path: [], + }, + }; + definitionCopy.assetKey.path = scenario.liveData + ? [scenario.liveData.stepKey] + : JSON.parse(scenario.definition.id); + + const dimensions = getAssetNodeDimensions(definitionCopy); + _setCacheEntryForTest(definitionCopy.assetKey, scenario.liveData); return (
{ overflowY: 'hidden', }} > - +
- +
@@ -79,6 +85,7 @@ export const PartnerTags = () => { const caseWithComputeKind = (computeKind: string) => { const def = {...Mocks.AssetNodeFragmentBasic, computeKind}; const liveData = Mocks.LiveDataForNodeMaterialized; + _setCacheEntryForTest(buildAssetKey({path: [liveData.stepKey]}), liveData); const dimensions = getAssetNodeDimensions(def); return ( @@ -87,7 +94,7 @@ export const PartnerTags = () => {
- +
); diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/__stories__/SidebarAssetInfo.stories.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/__stories__/SidebarAssetInfo.stories.tsx index 5dbbbfeadae6f..a0d60c119d07e 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/__stories__/SidebarAssetInfo.stories.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/__stories__/SidebarAssetInfo.stories.tsx @@ -19,7 +19,6 @@ import { import {WorkspaceProvider} from '../../workspace/WorkspaceContext'; import {SIDEBAR_ASSET_QUERY, SidebarAssetInfo} from '../SidebarAssetInfo'; import {GraphNode} from '../Utils'; -import {LiveDataForNodeMaterializedWithChecks} from '../__fixtures__/AssetNode.fixtures'; import {SidebarAssetQuery} from '../types/SidebarAssetInfo.types'; // eslint-disable-next-line import/no-default-export @@ -252,7 +251,7 @@ const TestContainer: React.FC<{ export const AssetWithMaterializations = () => { return ( - + ); }; @@ -284,7 +283,7 @@ export const AssetWithPolicies = () => { }), ]} > - + ); }; @@ -292,10 +291,7 @@ export const AssetWithPolicies = () => { export const AssetWithGraphName = () => { return ( - + ); }; @@ -303,10 +299,7 @@ export const AssetWithGraphName = () => { export const AssetWithAssetChecks = () => { return ( - + ); }; @@ -314,10 +307,7 @@ export const AssetWithAssetChecks = () => { export const AssetWithDifferentOpName = () => { return ( - + ); }; @@ -325,10 +315,7 @@ export const AssetWithDifferentOpName = () => { export const ObservableSourceAsset = () => { return ( - + ); }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/__tests__/AssetNode.test.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/__tests__/AssetNode.test.tsx index 5c740a225adcd..bcacc9b2f1d46 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/__tests__/AssetNode.test.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/__tests__/AssetNode.test.tsx @@ -2,6 +2,7 @@ import {render, screen, waitFor} from '@testing-library/react'; import * as React from 'react'; import {MemoryRouter} from 'react-router-dom'; +import {_setCacheEntryForTest} from '../../asset-data/AssetLiveDataProvider'; import {AssetNode} from '../AssetNode'; import { AssetNodeScenariosBase, @@ -18,18 +19,26 @@ const Scenarios = [ describe('AssetNode', () => { Scenarios.forEach((scenario) => it(`renders ${scenario.expectedText.join(',')} when ${scenario.title}`, async () => { + const definitionCopy = { + ...scenario.definition, + assetKey: { + ...scenario.definition.assetKey, + path: [], + }, + }; + definitionCopy.assetKey.path = scenario.liveData + ? [scenario.liveData.stepKey] + : JSON.parse(scenario.definition.id); + _setCacheEntryForTest(definitionCopy.assetKey, scenario.liveData); + render( - + , ); await waitFor(() => { - const assetKey = scenario.definition.assetKey; + const assetKey = definitionCopy.assetKey; const displayName = assetKey.path[assetKey.path.length - 1]!; expect(screen.getByText(displayName)).toBeVisible(); for (const text of scenario.expectedText) { diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeDefinition.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeDefinition.tsx index 2ccd23fc6f73d..499b151a2bfcb 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeDefinition.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeDefinition.tsx @@ -14,7 +14,7 @@ import {Link} from 'react-router-dom'; import {COMMON_COLLATOR} from '../app/Util'; import {ASSET_NODE_FRAGMENT} from '../asset-graph/AssetNode'; -import {isHiddenAssetGroupJob, LiveData, toGraphId} from '../asset-graph/Utils'; +import {isHiddenAssetGroupJob} from '../asset-graph/Utils'; import {AssetNodeForGraphQueryFragment} from '../asset-graph/types/useAssetGraphData.types'; import {DagsterTypeSummary} from '../dagstertype/DagsterType'; import {Description} from '../pipelines/Description'; @@ -46,11 +46,9 @@ export const AssetNodeDefinition: React.FC<{ assetNode: AssetNodeDefinitionFragment; upstream: AssetNodeForGraphQueryFragment[] | null; downstream: AssetNodeForGraphQueryFragment[] | null; - liveDataByNode: LiveData; dependsOnSelf: boolean; -}> = ({assetNode, upstream, downstream, liveDataByNode, dependsOnSelf}) => { +}> = ({assetNode, upstream, downstream, dependsOnSelf}) => { const {assetMetadata, assetType} = metadataForAssetNode(assetNode); - const liveDataForNode = liveDataByNode[toGraphId(assetNode.assetKey)]; const configType = assetNode.configField?.configType; const assetConfigSchema = configType && configType.key !== 'Any' ? configType : null; @@ -109,11 +107,7 @@ export const AssetNodeDefinition: React.FC<{ {freshnessPolicyDescription(assetNode.freshnessPolicy)} - + )} @@ -149,7 +143,7 @@ export const AssetNodeDefinition: React.FC<{ {dependsOnSelf && } - + - + {/** Ensures the line between the left and right columns goes to the bottom of the page */}
diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineage.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineage.tsx index 9cb8c0eab6f0e..d2b9bf75793e2 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineage.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineage.tsx @@ -90,12 +90,7 @@ export const AssetNodeLineage: React.FC<{ Not all upstream/downstream assets shown. Increase the depth to show more. )} - + ); }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineageGraph.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineageGraph.tsx index eb1a4db3cd2d1..2cf7d8c3c7228 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineageGraph.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineageGraph.tsx @@ -8,7 +8,7 @@ import {MINIMAL_SCALE} from '../asset-graph/AssetGraphExplorer'; import {AssetGroupNode} from '../asset-graph/AssetGroupNode'; import {AssetNodeMinimal, AssetNode} from '../asset-graph/AssetNode'; import {AssetNodeLink} from '../asset-graph/ForeignNode'; -import {GraphData, LiveData, toGraphId} from '../asset-graph/Utils'; +import {GraphData, toGraphId} from '../asset-graph/Utils'; import {DEFAULT_MAX_ZOOM, SVGViewport} from '../graph/SVGViewport'; import {useAssetLayout} from '../graph/asyncGraphLayout'; import {AssetKeyInput} from '../graphql/types'; @@ -22,9 +22,8 @@ const LINEAGE_GRAPH_ZOOM_LEVEL = 'lineageGraphZoomLevel'; export const AssetNodeLineageGraph: React.FC<{ assetKey: AssetKeyInput; assetGraphData: GraphData; - liveDataByNode: LiveData; params: AssetViewParams; -}> = ({assetKey, assetGraphData, liveDataByNode, params}) => { +}> = ({assetKey, assetGraphData, params}) => { const assetGraphId = toGraphId(assetKey); const [highlighted, setHighlighted] = React.useState(null); @@ -70,10 +69,10 @@ export const AssetNodeLineageGraph: React.FC<{ maxZoom={DEFAULT_MAX_ZOOM} maxAutocenterZoom={DEFAULT_MAX_ZOOM} > - {({scale}) => ( + {({scale}, viewportRect) => ( {viewportEl.current && } - + {Object.values(layout.groups) .sort((a, b) => a.id.length - b.id.length) @@ -105,13 +104,11 @@ export const AssetNodeLineageGraph: React.FC<{ ) : scale < MINIMAL_SCALE ? ( ) : ( )} diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeList.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeList.tsx index e81aa3a562ae6..d7219246a6bb2 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeList.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeList.tsx @@ -4,15 +4,13 @@ import {useHistory} from 'react-router-dom'; import styled from 'styled-components'; import {AssetNode} from '../asset-graph/AssetNode'; -import {LiveData, toGraphId} from '../asset-graph/Utils'; import {AssetNodeForGraphQueryFragment} from '../asset-graph/types/useAssetGraphData.types'; import {assetDetailsPathForKey} from './assetDetailsPathForKey'; export const AssetNodeList: React.FC<{ items: AssetNodeForGraphQueryFragment[] | null; - liveDataByNode: LiveData; -}> = ({items, liveDataByNode}) => { +}> = ({items}) => { const history = useHistory(); if (items === null) { @@ -33,11 +31,7 @@ export const AssetNodeList: React.FC<{ history.push(assetDetailsPathForKey(asset.assetKey, {view: 'definition'})); }} > - + ))} diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetSidebarActivitySummary.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetSidebarActivitySummary.tsx index 0f5e9d8fe97d5..d96a446aa472a 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetSidebarActivitySummary.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetSidebarActivitySummary.tsx @@ -71,11 +71,7 @@ export const AssetSidebarActivitySummary: React.FC = ({ {freshnessPolicyDescription(asset.freshnessPolicy)} - + )} diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx index 8a9d3e821166d..3c215683d24e8 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx @@ -118,7 +118,6 @@ export const AssetView = ({assetKey}: Props) => { upstream={upstream} downstream={downstream} dependsOnSelf={node ? nodeDependsOnSelf(node) : false} - liveDataByNode={liveDataByNode} /> ); }; @@ -487,11 +486,7 @@ const AssetViewPageHeaderTags: React.FC<{ )} {definition && definition.autoMaterializePolicy && } {definition && definition.freshnessPolicy && ( - + )} {definition && ( dayjs.duration(minLate, 'minutes').humanize(false); export const OverdueTag: React.FC<{ - liveData: LiveDataForNode | undefined; policy: Pick; assetKey: AssetKeyInput; -}> = ({liveData, policy, assetKey}) => { +}> = ({policy, assetKey}) => { + const liveData = useAssetLiveData(assetKey); + if (!liveData?.freshnessInfo) { return null; } diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/__stories__/AssetNodeDefinition.stories.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/__stories__/AssetNodeDefinition.stories.tsx index 236308e864e9e..f23e6ed0bdcc9 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/__stories__/AssetNodeDefinition.stories.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/__stories__/AssetNodeDefinition.stories.tsx @@ -30,7 +30,6 @@ export const MinimalAsset = () => { dependsOnSelf={false} downstream={[]} upstream={[]} - liveDataByNode={{}} assetNode={ buildAssetNode({ description: null, @@ -56,7 +55,6 @@ export const FullUseAsset = () => { buildAssetNode({assetKey: buildAssetKey({path: ['downstream_1']})}), buildAssetNode({assetKey: buildAssetKey({path: ['downstream_2']})}), ]} - liveDataByNode={{}} assetNode={ buildAssetNode({ description: ` diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/__stories__/OverdueTag.stories.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/__stories__/OverdueTag.stories.tsx index a25f43cf9d459..f2ffbd94cd087 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/__stories__/OverdueTag.stories.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/__stories__/OverdueTag.stories.tsx @@ -3,10 +3,6 @@ import {Box} from '@dagster-io/ui-components'; import React from 'react'; import {createAppCache} from '../../app/AppCache'; -import { - LiveDataForNodeMaterialized, - LiveDataForNodeMaterializedAndOverdue, -} from '../../asset-graph/__fixtures__/AssetNode.fixtures'; import { AssetFreshnessInfo, FreshnessPolicy, @@ -29,13 +25,13 @@ const TEST_LAG_MINUTES = 4 * 60 + 30; const TEST_TIME = Date.now(); const LAST_MATERIALIZATION_TIME = TEST_TIME - 4 * 60 * 1000; -const mockLiveData = { - ...LiveDataForNodeMaterializedAndOverdue, - lastMaterialization: { - ...LiveDataForNodeMaterializedAndOverdue.lastMaterialization!, - timestamp: `${LAST_MATERIALIZATION_TIME}`, - }, -}; +// const mockLiveData = { +// ...LiveDataForNodeMaterializedAndOverdue, +// lastMaterialization: { +// ...LiveDataForNodeMaterializedAndOverdue.lastMaterialization!, +// timestamp: `${LAST_MATERIALIZATION_TIME}`, +// }, +// }; function buildOverduePopoverMock( policy: FreshnessPolicy, @@ -105,11 +101,7 @@ export const OverdueCronSchedule = () => { mocks={[buildOverduePopoverMock(mockFreshnessPolicyCron, freshnessInfo)]} > - + Hover for details, times are relative to last cron tick (eg: earlier). Note: The relative materialization times in the modal are not mocked to align with the cron schedule tick. @@ -136,11 +128,7 @@ export const OverdueNoSchedule = () => { mocks={[buildOverduePopoverMock(mockFreshnessPolicy, freshnessInfo)]} > - + {' Hover for details, times are relative to now (eg: "ago")'} @@ -168,11 +156,7 @@ export const OverdueNoUpstreams = () => { mocks={[buildOverduePopoverMock(mockFreshnessPolicy, freshnessInfo, false)]} > - + {' Hover for details. "derived from upstream data" omitted from description.'} @@ -186,19 +170,15 @@ export const NeverMaterialized = () => { maximumLagMinutes: 24 * 60, lastEvaluationTimestamp: `${TEST_TIME}`, }); - const freshnessInfo = { - __typename: 'AssetFreshnessInfo' as const, - currentMinutesLate: null, - currentLagMinutes: null, - }; + // const freshnessInfo = { + // __typename: 'AssetFreshnessInfo' as const, + // currentMinutesLate: null, + // currentLagMinutes: null, + // }; return ( - + ); }; @@ -221,11 +201,7 @@ export const Fresh = () => { cache={createAppCache()} mocks={[buildOverduePopoverMock(mockFreshnessPolicyMet, freshnessInfo)]} > - + ); }; @@ -239,11 +215,7 @@ export const NoFreshnessInfo = () => { }); return ( - + ); }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/overview/OverviewAssetsRoot.tsx b/js_modules/dagster-ui/packages/ui-core/src/overview/OverviewAssetsRoot.tsx index 234f9d451b627..55f3a0d134729 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/overview/OverviewAssetsRoot.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/overview/OverviewAssetsRoot.tsx @@ -223,7 +223,7 @@ function VirtualRow({height, start, group}: RowProps) { [group.assets], ); - const liveDataByNode = useAssetsLiveData(assetKeys); + const {liveDataByNode} = useAssetsLiveData(assetKeys); const statuses = React.useMemo(() => { type assetType = (typeof group)['assets'][0];