From b0bcb20192ac24218a2f1b29fbdcd2cdac16b801 Mon Sep 17 00:00:00 2001 From: Marco polo Date: Mon, 9 Dec 2024 13:31:09 -0500 Subject: [PATCH] Move asset selection filtering to webworker (#26315) ## Summary & Motivation Move our heavy asset selection filtering to a webworker. Added a feature flag as an escape hatch. ## How I Tested These Changes Tested primarily with cloud-proxy with customers that have large graphs. --- .../jest/mocks/ComputeGraphData.worker.ts | 36 ++++ .../dagster-ui/packages/ui-core/package.json | 4 + .../src/app/DefaultFeatureFlags.oss.tsx | 2 + .../ui-core/src/app/FeatureFlags.oss.tsx | 1 + .../packages/ui-core/src/app/Flags.tsx | 12 +- .../packages/ui-core/src/app/Util.tsx | 26 +-- .../__tests__/AssetLiveDataProvider.test.tsx | 13 ++ .../src/asset-graph/AssetGraphExplorer.tsx | 41 +++-- .../src/asset-graph/ComputeGraphData.ts | 79 +++++++++ .../src/asset-graph/ComputeGraphData.types.ts | 12 ++ .../asset-graph/ComputeGraphData.worker.ts | 25 +++ .../ui-core/src/asset-graph/Utils.tsx | 2 +- .../src/asset-graph/useAssetGraphData.tsx | 163 ++++++++++-------- .../useAssetSelectionFiltering.tsx | 21 ++- .../useAssetSelectionInput.tsx | 6 +- .../ui-core/src/assets/AssetGroupRoot.tsx | 2 +- .../ui-core/src/assets/AssetsCatalogTable.tsx | 10 +- .../src/assets/__tests__/AssetView.test.tsx | 15 +- .../useAssetDefinitionFilterState.oss.tsx | 3 +- .../src/automation/MergedAutomationRoot.tsx | 2 +- .../ui-core/src/graph/asyncGraphLayout.ts | 4 +- .../live-data-provider/LiveDataScheduler.tsx | 36 ++++ .../src/live-data-provider/LiveDataThread.tsx | 34 ++-- .../LiveDataThreadManager.tsx | 2 +- .../packages/ui-core/src/setupTests.ts | 3 + .../doesFilterArrayMatchValueArray.tsx | 25 +++ .../src/ui/Filters/useDefinitionTagFilter.tsx | 18 -- .../util/__tests__/generateObjectHash.test.ts | 99 +++++++++++ .../ui-core/src/util/generateObjectHash.ts | 103 +++++++++++ js_modules/dagster-ui/yarn.lock | 32 ++++ 30 files changed, 681 insertions(+), 150 deletions(-) create mode 100644 js_modules/dagster-ui/packages/ui-core/jest/mocks/ComputeGraphData.worker.ts create mode 100644 js_modules/dagster-ui/packages/ui-core/src/asset-graph/ComputeGraphData.ts create mode 100644 js_modules/dagster-ui/packages/ui-core/src/asset-graph/ComputeGraphData.types.ts create mode 100644 js_modules/dagster-ui/packages/ui-core/src/asset-graph/ComputeGraphData.worker.ts create mode 100644 js_modules/dagster-ui/packages/ui-core/src/live-data-provider/LiveDataScheduler.tsx create mode 100644 js_modules/dagster-ui/packages/ui-core/src/ui/Filters/doesFilterArrayMatchValueArray.tsx create mode 100644 js_modules/dagster-ui/packages/ui-core/src/util/__tests__/generateObjectHash.test.ts create mode 100644 js_modules/dagster-ui/packages/ui-core/src/util/generateObjectHash.ts diff --git a/js_modules/dagster-ui/packages/ui-core/jest/mocks/ComputeGraphData.worker.ts b/js_modules/dagster-ui/packages/ui-core/jest/mocks/ComputeGraphData.worker.ts new file mode 100644 index 0000000000000..f7543cad6d2c9 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/jest/mocks/ComputeGraphData.worker.ts @@ -0,0 +1,36 @@ +import {setFeatureFlagsInternal} from '../../src/app/Flags'; +import {computeGraphData} from '../../src/asset-graph/ComputeGraphData'; +import {ComputeGraphDataMessageType} from '../../src/asset-graph/ComputeGraphData.types'; + +// eslint-disable-next-line import/no-default-export +export default class MockWorker { + onmessage = (_: any) => {}; + + addEventListener(_type: string, handler: any) { + this.onmessage = handler; + } + + // mock expects data: { } instead of e: { data: { } } + async postMessage(data: ComputeGraphDataMessageType) { + if (data.type === 'computeGraphData') { + if (data.flagAssetSelectionSyntax) { + setFeatureFlagsInternal({flagAssetSelectionSyntax: true}); + } + const state = await computeGraphData(data); + this.onmessage({data: state}); + } + } +} + +const originalWorker = global.Worker; +// @ts-expect-error - test shenanigans +global.Worker = function ComputeGraphDataMockWorkerWrapper( + url: string | URL, + opts?: WorkerOptions, +) { + if (url.toString().endsWith('ComputeGraphData.worker')) { + return new MockWorker(); + } else { + return new originalWorker(url, opts); + } +}; diff --git a/js_modules/dagster-ui/packages/ui-core/package.json b/js_modules/dagster-ui/packages/ui-core/package.json index 23d40dd9517ab..a17017e2ce43d 100644 --- a/js_modules/dagster-ui/packages/ui-core/package.json +++ b/js_modules/dagster-ui/packages/ui-core/package.json @@ -53,6 +53,7 @@ "dayjs": "^1.11.7", "deepmerge": "^4.2.2", "fake-indexeddb": "^4.0.2", + "fast-text-encoding": "^1.0.6", "fuse.js": "^6.4.2", "graphql": "^16.8.1", "graphql-codegen-persisted-query-ids": "^0.2.0", @@ -71,6 +72,7 @@ "rehype-sanitize": "^5.0.1", "remark": "^14.0.2", "remark-gfm": "3.0.1", + "spark-md5": "^3.0.2", "strip-markdown": "^6.0.0", "subscriptions-transport-ws": "^0.9.15", "worker-loader": "^3.0.8", @@ -117,6 +119,7 @@ "@types/color": "^3.0.2", "@types/dagre": "^0.7.42", "@types/faker": "^5.1.7", + "@types/fast-text-encoding": "^1.0.3", "@types/graphql": "^14.5.0", "@types/jest": "^29.5.11", "@types/lodash": "^4.14.145", @@ -127,6 +130,7 @@ "@types/react-dom": "^18.3.1", "@types/react-router": "^5.1.17", "@types/react-router-dom": "^5.3.3", + "@types/spark-md5": "^3", "@types/testing-library__jest-dom": "^5.14.2", "@types/ws": "^6.0.3", "@typescript-eslint/eslint-plugin": "^8.9.0", diff --git a/js_modules/dagster-ui/packages/ui-core/src/app/DefaultFeatureFlags.oss.tsx b/js_modules/dagster-ui/packages/ui-core/src/app/DefaultFeatureFlags.oss.tsx index cf5ec46efc12c..20590e2f50c18 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/app/DefaultFeatureFlags.oss.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/app/DefaultFeatureFlags.oss.tsx @@ -4,6 +4,8 @@ import {FeatureFlag} from 'shared/app/FeatureFlags.oss'; * Default values for feature flags when they are unset. */ export const DEFAULT_FEATURE_FLAG_VALUES: Partial> = { + [FeatureFlag.flagAssetSelectionWorker]: true, + // Flags for tests [FeatureFlag.__TestFlagDefaultTrue]: true, [FeatureFlag.__TestFlagDefaultFalse]: false, diff --git a/js_modules/dagster-ui/packages/ui-core/src/app/FeatureFlags.oss.tsx b/js_modules/dagster-ui/packages/ui-core/src/app/FeatureFlags.oss.tsx index 5a4a7512fb851..c9244178d1cfe 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/app/FeatureFlags.oss.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/app/FeatureFlags.oss.tsx @@ -6,6 +6,7 @@ export enum FeatureFlag { flagLegacyRunsPage = 'flagLegacyRunsPage', flagAssetSelectionSyntax = 'flagAssetSelectionSyntax', flagRunSelectionSyntax = 'flagRunSelectionSyntax', + flagAssetSelectionWorker = 'flagAssetSelectionWorker', // Flags for tests __TestFlagDefaultNone = '__TestFlagDefaultNone', diff --git a/js_modules/dagster-ui/packages/ui-core/src/app/Flags.tsx b/js_modules/dagster-ui/packages/ui-core/src/app/Flags.tsx index ac9d1eed0ed50..978e1e2949c34 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/app/Flags.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/app/Flags.tsx @@ -28,7 +28,7 @@ const initializeFeatureFlags = () => { flags.forEach((flag: FeatureFlag) => { migratedFlags[flag] = true; }); - setFeatureFlagsInternal(migratedFlags, false); // Prevent broadcasting during migration + setFeatureFlagsInternal(migratedFlags); flags = migratedFlags; } @@ -37,16 +37,15 @@ const initializeFeatureFlags = () => { /** * Internal function to set feature flags without broadcasting. - * Used during initialization and migration. + * Used during initialization and migration and by web-workers. */ -const setFeatureFlagsInternal = (flags: FeatureFlagMap, broadcast: boolean = true) => { +export const setFeatureFlagsInternal = (flags: FeatureFlagMap) => { if (typeof flags !== 'object' || Array.isArray(flags)) { throw new Error('flags must be an object mapping FeatureFlag to boolean values'); } currentFeatureFlags = flags; - localStorage.setItem(DAGSTER_FLAGS_KEY, JSON.stringify(flags)); - if (broadcast) { - featureFlagsChannel.postMessage('updated'); + if (typeof localStorage !== 'undefined') { + localStorage.setItem(DAGSTER_FLAGS_KEY, JSON.stringify(flags)); } }; @@ -128,4 +127,5 @@ export const useFeatureFlags = (): Readonly> => { */ export const setFeatureFlags = (flags: FeatureFlagMap) => { setFeatureFlagsInternal(flags); + featureFlagsChannel.postMessage('updated'); }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/app/Util.tsx b/js_modules/dagster-ui/packages/ui-core/src/app/Util.tsx index 663f422208a57..f09f482ea1555 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/app/Util.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/app/Util.tsx @@ -134,7 +134,7 @@ export function asyncMemoize Promise hashFn?: (arg: T, ...rest: any[]) => any, hashSize?: number, ): U { - const cache = new LRU(hashSize || 50); + const cache = new LRU(hashSize || 50); return (async (arg: T, ...rest: any[]) => { const key = hashFn ? hashFn(arg, ...rest) : arg; if (cache.has(key)) { @@ -160,6 +160,8 @@ export function indexedDBAsyncMemoize> = {}; + async function genHashKey(arg: T, ...rest: any[]) { const hash = hashFn ? hashFn(arg, ...rest) : arg; @@ -182,17 +184,21 @@ export function indexedDBAsyncMemoize { + const result = await fn(arg, ...rest); + // Resolve the promise before storing the result in IndexedDB + res(result); + if (lru) { + await lru.set(hashKey, result, { + // Some day in the year 2050... + expiry: new Date(9 ** 13), + }); + delete hashToPromise[hashKey]; + } }); } + resolve(await hashToPromise[hashKey]!); }); }) as any; ret.isCached = async (arg: T, ...rest: any) => { diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-data/__tests__/AssetLiveDataProvider.test.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-data/__tests__/AssetLiveDataProvider.test.tsx index 2d8b826bebb17..5a1964a8f44bf 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-data/__tests__/AssetLiveDataProvider.test.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-data/__tests__/AssetLiveDataProvider.test.tsx @@ -30,6 +30,19 @@ jest.mock('../../live-data-provider/util', () => { }; }); +jest.mock('../../live-data-provider/LiveDataScheduler', () => { + return { + LiveDataScheduler: class LiveDataScheduler { + scheduleStartFetchLoop(doStart: () => void) { + doStart(); + } + scheduleStopFetchLoop(doStop: () => void) { + doStop(); + } + }, + }; +}); + function Test({ mocks, hooks, diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx index ed250aacd5f0d..76643fac97872 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx @@ -98,15 +98,24 @@ type Props = { export const MINIMAL_SCALE = 0.6; export const GROUPS_ONLY_SCALE = 0.15; +const DEFAULT_SET_HIDE_NODES_MATCH = (_node: AssetNodeForGraphQueryFragment) => true; + export const AssetGraphExplorer = (props: Props) => { const fullAssetGraphData = useFullAssetGraphData(props.fetchOptions); - const [hideNodesMatching, setHideNodesMatching] = useState( - () => (_node: AssetNodeForGraphQueryFragment) => true, - ); + const [hideNodesMatching, setHideNodesMatching] = useState(() => DEFAULT_SET_HIDE_NODES_MATCH); - const {fetchResult, assetGraphData, graphQueryItems, allAssetKeys} = useAssetGraphData( + const { + loading: graphDataLoading, + fetchResult, + assetGraphData, + graphQueryItems, + allAssetKeys, + } = useAssetGraphData( props.explorerPath.opsQuery, - {...props.fetchOptions, hideNodesMatching}, + useMemo( + () => ({...props.fetchOptions, hideNodesMatching}), + [props.fetchOptions, hideNodesMatching], + ), ); const {explorerPath, onChangeExplorerPath} = props; @@ -119,7 +128,7 @@ export const AssetGraphExplorer = (props: Props) => { () => (fullAssetGraphData ? Object.values(fullAssetGraphData.nodes) : []), [fullAssetGraphData], ), - loading: fetchResult.loading, + loading: graphDataLoading, viewType: props.viewType, assetSelection: explorerPath.opsQuery, setAssetSelection: React.useCallback( @@ -169,7 +178,7 @@ export const AssetGraphExplorer = (props: Props) => { filterButton={button} kindFilter={kindFilter} groupsFilter={groupsFilter} - filteredAssetsLoading={filteredAssetsLoading} + loading={filteredAssetsLoading || graphDataLoading} {...props} /> ); @@ -183,7 +192,7 @@ type WithDataProps = Props & { assetGraphData: GraphData; fullAssetGraphData: GraphData; graphQueryItems: AssetGraphQueryItem[]; - filteredAssetsLoading: boolean; + loading: boolean; filterButton: React.ReactNode; filterBar: React.ReactNode; @@ -209,7 +218,7 @@ const AssetGraphExplorerWithData = ({ viewType, kindFilter, groupsFilter, - filteredAssetsLoading, + loading: dataLoading, }: WithDataProps) => { const findAssetLocation = useFindAssetLocation(); const [highlighted, setHighlighted] = React.useState(null); @@ -235,7 +244,11 @@ const AssetGraphExplorerWithData = ({ }); const focusGroupIdAfterLayoutRef = React.useRef(''); - const {layout, loading, async} = useAssetLayout( + const { + layout, + loading: layoutLoading, + async, + } = useAssetLayout( assetGraphData, expandedGroups, useMemo(() => ({direction}), [direction]), @@ -665,6 +678,8 @@ const AssetGraphExplorerWithData = ({ ) : null; + const loading = layoutLoading || dataLoading; + const explorer = ( Loading assets… @@ -769,7 +784,7 @@ const AssetGraphExplorerWithData = ({ ) } second={ - filteredAssetsLoading ? null : selectedGraphNodes.length === 1 && selectedGraphNodes[0] ? ( + loading ? null : selectedGraphNodes.length === 1 && selectedGraphNodes[0] ? ( @@ -814,7 +829,7 @@ const AssetGraphExplorerWithData = ({ setShowSidebar(false); }} onFilterToGroup={onFilterToGroup} - loading={filteredAssetsLoading} + loading={loading} /> } second={explorer} diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/ComputeGraphData.ts b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/ComputeGraphData.ts new file mode 100644 index 0000000000000..988454044b5bb --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/ComputeGraphData.ts @@ -0,0 +1,79 @@ +import groupBy from 'lodash/groupBy'; + +import {ComputeGraphDataMessageType} from './ComputeGraphData.types'; +import {GraphData, buildGraphData, toGraphId} from './Utils'; +import {AssetNodeForGraphQueryFragment} from './types/useAssetGraphData.types'; +import {GraphDataState} from './useAssetGraphData'; +import {filterAssetSelectionByQuery} from '../asset-selection/AntlrAssetSelection'; +import {doesFilterArrayMatchValueArray} from '../ui/Filters/doesFilterArrayMatchValueArray'; + +export function computeGraphData({ + repoFilteredNodes, + graphQueryItems, + opsQuery, + kinds: _kinds, + hideEdgesToNodesOutsideQuery, +}: Omit): GraphDataState { + if (repoFilteredNodes === undefined || graphQueryItems === undefined) { + return { + allAssetKeys: [], + graphAssetKeys: [], + assetGraphData: null, + }; + } + + // Filter the set of all AssetNodes down to those matching the `opsQuery`. + // In the future it might be ideal to move this server-side, but we currently + // get to leverage the useQuery cache almost 100% of the time above, making this + // super fast after the first load vs a network fetch on every page view. + const {all: allFilteredByOpQuery} = filterAssetSelectionByQuery(graphQueryItems, opsQuery); + const kinds = _kinds?.map((c) => c.toLowerCase()); + const all = kinds?.length + ? allFilteredByOpQuery.filter( + ({node}) => + node.kinds && + doesFilterArrayMatchValueArray( + kinds, + node.kinds.map((k) => k.toLowerCase()), + ), + ) + : allFilteredByOpQuery; + + // Assemble the response into the data structure used for layout, traversal, etc. + const assetGraphData = buildGraphData(all.map((n) => n.node)); + if (hideEdgesToNodesOutsideQuery) { + removeEdgesToHiddenAssets(assetGraphData, repoFilteredNodes); + } + + return { + allAssetKeys: repoFilteredNodes.map((n) => n.assetKey), + graphAssetKeys: all.map((n) => ({path: n.node.assetKey.path})), + assetGraphData, + }; +} + +const removeEdgesToHiddenAssets = ( + graphData: GraphData, + allNodes: AssetNodeForGraphQueryFragment[], +) => { + const allNodesById = groupBy(allNodes, (n) => toGraphId(n.assetKey)); + const notSourceAsset = (id: string) => !!allNodesById[id]; + + for (const node of Object.keys(graphData.upstream)) { + for (const edge of Object.keys(graphData.upstream[node]!)) { + if (!graphData.nodes[edge] && notSourceAsset(node)) { + delete graphData.upstream[node]![edge]; + delete graphData.downstream[edge]![node]; + } + } + } + + for (const node of Object.keys(graphData.downstream)) { + for (const edge of Object.keys(graphData.downstream[node]!)) { + if (!graphData.nodes[edge] && notSourceAsset(node)) { + delete graphData.upstream[edge]![node]; + delete graphData.downstream[node]![edge]; + } + } + } +}; diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/ComputeGraphData.types.ts b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/ComputeGraphData.types.ts new file mode 100644 index 0000000000000..b64e685d2414e --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/ComputeGraphData.types.ts @@ -0,0 +1,12 @@ +import {AssetNodeForGraphQueryFragment} from './types/useAssetGraphData.types'; +import {AssetGraphFetchScope, AssetGraphQueryItem} from './useAssetGraphData'; + +export type ComputeGraphDataMessageType = { + type: 'computeGraphData'; + repoFilteredNodes?: AssetNodeForGraphQueryFragment[]; + graphQueryItems?: AssetGraphQueryItem[]; + opsQuery: string; + kinds: AssetGraphFetchScope['kinds']; + hideEdgesToNodesOutsideQuery?: boolean; + flagAssetSelectionSyntax?: boolean; +}; diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/ComputeGraphData.worker.ts b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/ComputeGraphData.worker.ts new file mode 100644 index 0000000000000..0fae9ac4d1839 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/ComputeGraphData.worker.ts @@ -0,0 +1,25 @@ +import {FeatureFlag} from 'shared/app/FeatureFlags.oss'; + +import {computeGraphData} from './ComputeGraphData'; +import {ComputeGraphDataMessageType} from './ComputeGraphData.types'; +import {setFeatureFlags} from '../app/Flags'; + +type WorkerMessageData = ComputeGraphDataMessageType; + +self.addEventListener('message', async (event: MessageEvent & {data: WorkerMessageData}) => { + const {data} = event; + + if (data.type === 'computeGraphData') { + if (data.flagAssetSelectionSyntax) { + setFeatureFlags({[FeatureFlag.flagAssetSelectionSyntax]: true}); + } + const state = await computeGraphData(data); + self.postMessage(state); + } +}); + +self.onmessage = function (event) { + if (event.data === 'close') { + self.close(); // Terminates the worker + } +}; diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/Utils.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/Utils.tsx index 44ac79b147eab..74757d22d1cf8 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/Utils.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/Utils.tsx @@ -132,7 +132,7 @@ export const graphHasCycles = (graphData: GraphData) => { }; let hasCycles = false; while (nodes.size !== 0 && !hasCycles) { - hasCycles = search([], nodes.values().next().value); + hasCycles = search([], nodes.values().next().value!); } return hasCycles; }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/useAssetGraphData.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/useAssetGraphData.tsx index b30668c72bc5a..cea5968791a6a 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/useAssetGraphData.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/useAssetGraphData.tsx @@ -1,11 +1,16 @@ -import groupBy from 'lodash/groupBy'; import keyBy from 'lodash/keyBy'; +import memoize from 'lodash/memoize'; import reject from 'lodash/reject'; -import {useMemo} from 'react'; +import throttle from 'lodash/throttle'; +import {useEffect, useMemo, useRef, useState} from 'react'; +import {FeatureFlag} from 'shared/app/FeatureFlags.oss'; import {ASSET_NODE_FRAGMENT} from './AssetNode'; -import {GraphData, buildGraphData, toGraphId, tokenForAssetKey} from './Utils'; +import {GraphData, buildGraphData, tokenForAssetKey} from './Utils'; import {gql} from '../apollo-client'; +import {computeGraphData as computeGraphDataImpl} from './ComputeGraphData'; +import {ComputeGraphDataMessageType} from './ComputeGraphData.types'; +import {featureEnabled} from '../app/Flags'; import { AssetGraphQuery, AssetGraphQueryVariables, @@ -14,11 +19,11 @@ import { } from './types/useAssetGraphData.types'; import {usePrefixedCacheKey} from '../app/AppProvider'; import {GraphQueryItem} from '../app/GraphQueryImpl'; -import {filterAssetSelectionByQuery} from '../asset-selection/AntlrAssetSelection'; +import {indexedDBAsyncMemoize} from '../app/Util'; import {AssetKey} from '../assets/types'; import {AssetGroupSelector, PipelineSelector} from '../graphql/types'; +import {useBlockTraceUntilTrue} from '../performance/TraceContext'; import {useIndexedDBCachedQuery} from '../search/useIndexedDBCachedQuery'; -import {doesFilterArrayMatchValueArray} from '../ui/Filters/useDefinitionTagFilter'; export interface AssetGraphFetchScope { hideEdgesToNodesOutsideQuery?: boolean; @@ -26,6 +31,10 @@ export interface AssetGraphFetchScope { pipelineSelector?: PipelineSelector; groupSelector?: AssetGroupSelector; kinds?: string[]; + + // This is used to indicate we shouldn't start handling any input. + // This is used by pages where `hideNodesMatching` is only available asynchronously. + loading?: boolean; } export type AssetGraphQueryItem = GraphQueryItem & { @@ -61,6 +70,17 @@ export function useFullAssetGraphData(options: AssetGraphFetchScope) { return fullAssetGraphData; } +export type GraphDataState = { + graphAssetKeys: AssetKey[]; + allAssetKeys: AssetKey[]; + assetGraphData: GraphData | null; +}; +const INITIAL_STATE: GraphDataState = { + graphAssetKeys: [], + allAssetKeys: [], + assetGraphData: null, +}; + /** Fetches data for rendering an asset graph: * * @param pipelineSelector: Optionally scope to an asset job, or pass null for the global graph @@ -105,58 +125,55 @@ export function useAssetGraphData(opsQuery: string, options: AssetGraphFetchScop [repoFilteredNodes], ); - const {assetGraphData, graphAssetKeys, allAssetKeys} = useMemo(() => { - if (repoFilteredNodes === undefined || graphQueryItems === undefined) { - return { - graphAssetKeys: [], - graphQueryItems: [], - assetGraphData: null, - }; - } + const [state, setState] = useState(INITIAL_STATE); - // Filter the set of all AssetNodes down to those matching the `opsQuery`. - // In the future it might be ideal to move this server-side, but we currently - // get to leverage the useQuery cache almost 100% of the time above, making this - // super fast after the first load vs a network fetch on every page view. - const {all: allFilteredByOpQuery} = filterAssetSelectionByQuery(graphQueryItems, opsQuery); - const kinds = options.kinds?.map((c) => c.toLowerCase()); - const all = kinds?.length - ? allFilteredByOpQuery.filter( - ({node}) => - node.kinds && - doesFilterArrayMatchValueArray( - kinds, - node.kinds.map((k) => k.toLowerCase()), - ), - ) - : allFilteredByOpQuery; - - // Assemble the response into the data structure used for layout, traversal, etc. - const assetGraphData = buildGraphData(all.map((n) => n.node)); - if (options.hideEdgesToNodesOutsideQuery) { - removeEdgesToHiddenAssets(assetGraphData, repoFilteredNodes); - } + const {kinds, hideEdgesToNodesOutsideQuery} = options; - return { - allAssetKeys: repoFilteredNodes.map((n) => n.assetKey), - graphAssetKeys: all.map((n) => ({path: n.node.assetKey.path})), - assetGraphData, + const [graphDataLoading, setGraphDataLoading] = useState(true); + + const lastProcessedRequestRef = useRef(0); + const currentRequestRef = useRef(0); + + useEffect(() => { + if (options.loading) { + return; + } + const requestId = ++currentRequestRef.current; + setGraphDataLoading(true); + computeGraphData({ + repoFilteredNodes, graphQueryItems, - }; + opsQuery, + kinds, + hideEdgesToNodesOutsideQuery, + flagAssetSelectionSyntax: featureEnabled(FeatureFlag.flagAssetSelectionSyntax), + })?.then((data) => { + if (lastProcessedRequestRef.current < requestId) { + lastProcessedRequestRef.current = requestId; + setState(data); + if (requestId === currentRequestRef.current) { + setGraphDataLoading(false); + } + } + }); }, [ repoFilteredNodes, graphQueryItems, opsQuery, - options.kinds, - options.hideEdgesToNodesOutsideQuery, + kinds, + hideEdgesToNodesOutsideQuery, + options.loading, ]); + const loading = fetchResult.loading || graphDataLoading; + useBlockTraceUntilTrue('useAssetGraphData', !loading); return { + loading, fetchResult, - assetGraphData, + assetGraphData: state.assetGraphData, graphQueryItems, - graphAssetKeys, - allAssetKeys, + graphAssetKeys: state.graphAssetKeys, + allAssetKeys: state.allAssetKeys, }; } @@ -181,29 +198,6 @@ const buildGraphQueryItems = (nodes: AssetNode[]) => { return Object.values(items); }; -const removeEdgesToHiddenAssets = (graphData: GraphData, allNodes: AssetNode[]) => { - const allNodesById = groupBy(allNodes, (n) => toGraphId(n.assetKey)); - const notSourceAsset = (id: string) => !!allNodesById[id]; - - for (const node of Object.keys(graphData.upstream)) { - for (const edge of Object.keys(graphData.upstream[node]!)) { - if (!graphData.nodes[edge] && notSourceAsset(node)) { - delete graphData.upstream[node]![edge]; - delete graphData.downstream[edge]![node]; - } - } - } - - for (const node of Object.keys(graphData.downstream)) { - for (const edge of Object.keys(graphData.downstream[node]!)) { - if (!graphData.nodes[edge] && notSourceAsset(node)) { - delete graphData.upstream[edge]![node]; - delete graphData.downstream[node]![edge]; - } - } - } -}; - export const calculateGraphDistances = (items: GraphQueryItem[], assetKey: AssetKey) => { const map = keyBy(items, (g) => g.name); const start = map[tokenForAssetKey(assetKey)]; @@ -304,3 +298,36 @@ export const ASSET_GRAPH_QUERY = gql` ${ASSET_NODE_FRAGMENT} `; + +const computeGraphData = throttle( + indexedDBAsyncMemoize< + ComputeGraphDataMessageType, + GraphDataState, + typeof computeGraphDataWrapper + >(computeGraphDataWrapper, (props) => { + return JSON.stringify(props); + }), + 2000, + {leading: true}, +); + +const getWorker = memoize(() => new Worker(new URL('./ComputeGraphData.worker', import.meta.url))); + +async function computeGraphDataWrapper( + props: Omit, +): Promise { + if (featureEnabled(FeatureFlag.flagAssetSelectionWorker)) { + const worker = getWorker(); + return new Promise((resolve) => { + worker.addEventListener('message', (event) => { + resolve(event.data as GraphDataState); + }); + const message: ComputeGraphDataMessageType = { + type: 'computeGraphData', + ...props, + }; + worker.postMessage(message); + }); + } + return computeGraphDataImpl(props); +} diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-selection/useAssetSelectionFiltering.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-selection/useAssetSelectionFiltering.tsx index bcf797b8e5323..564240d2c8eeb 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-selection/useAssetSelectionFiltering.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-selection/useAssetSelectionFiltering.tsx @@ -13,9 +13,11 @@ export const useAssetSelectionFiltering = < definition?: FilterableAssetDefinition | null; }, >({ + loading: assetsLoading, assetSelection, assets, }: { + loading?: boolean; assetSelection: string; assets: T[]; @@ -25,7 +27,8 @@ export const useAssetSelectionFiltering = < [assets], ); - const {fetchResult, graphQueryItems, graphAssetKeys} = useAssetGraphData( + const assetsByKeyStringified = useMemo(() => JSON.stringify(assetsByKey), [assetsByKey]); + const {loading, graphQueryItems, graphAssetKeys} = useAssetGraphData( assetSelection, useMemo( () => ({ @@ -33,23 +36,31 @@ export const useAssetSelectionFiltering = < hideNodesMatching: (node: AssetNodeForGraphQueryFragment) => { return !assetsByKey[tokenForAssetKey(node.assetKey)]; }, + loading: !!assetsLoading, }), - [assetsByKey], + // eslint-disable-next-line react-hooks/exhaustive-deps + [assetsByKeyStringified, assetsLoading], ), ); const filtered = useMemo(() => { + if (!assetSelection) { + return assets; + } return ( graphAssetKeys - .map((key) => assetsByKey[tokenForAssetKey(key)]!) + .map((key) => { + return assetsByKey[tokenForAssetKey(key)]!; + }) + .filter((a) => a) .sort((a, b) => COMMON_COLLATOR.compare(a.key.path.join(''), b.key.path.join(''))) ?? [] ); - }, [graphAssetKeys, assetsByKey]); + }, [assetSelection, graphAssetKeys, assets, assetsByKey]); const filteredByKey = useMemo( () => Object.fromEntries(filtered.map((asset) => [tokenForAssetKey(asset.key), asset])), [filtered], ); - return {filtered, filteredByKey, fetchResult, graphAssetKeys, graphQueryItems}; + return {filtered, filteredByKey, loading, graphAssetKeys, graphQueryItems}; }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-selection/useAssetSelectionInput.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-selection/useAssetSelectionInput.tsx index 31e5dc83fc717..70bfc494821a5 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-selection/useAssetSelectionInput.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-selection/useAssetSelectionInput.tsx @@ -15,12 +15,14 @@ export const useAssetSelectionInput = < }, >( assets: T[], + assetsLoading?: boolean, ) => { const [assetSelection, setAssetSelection] = useAssetSelectionState(); - const {graphQueryItems, fetchResult, filtered} = useAssetSelectionFiltering({ + const {graphQueryItems, loading, filtered} = useAssetSelectionFiltering({ assetSelection, assets, + loading: !!assetsLoading, }); let filterInput = ( @@ -43,5 +45,5 @@ export const useAssetSelectionInput = < ); } - return {filterInput, fetchResult, filtered, assetSelection, setAssetSelection}; + return {filterInput, loading, filtered, assetSelection, setAssetSelection}; }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetGroupRoot.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetGroupRoot.tsx index 2e4a8fe8ccfc4..51c1544fbc3df 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetGroupRoot.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetGroupRoot.tsx @@ -93,7 +93,7 @@ export const AssetGroupRoot = ({ [history, openInNewTab], ); - const fetchOptions = React.useMemo(() => ({groupSelector}), [groupSelector]); + const fetchOptions = React.useMemo(() => ({groupSelector, loading: false}), [groupSelector]); const lineageOptions = React.useMemo( () => ({preferAssetRendering: true, explodeComposites: true}), diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetsCatalogTable.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetsCatalogTable.tsx index 040f0d3dfe570..5265eaae1d4d6 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetsCatalogTable.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetsCatalogTable.tsx @@ -210,10 +210,10 @@ export const AssetsCatalogTable = ({ activeFiltersJsx, kindFilter, } = useAssetCatalogFiltering({assets}); - const {filterInput, filtered, fetchResult, assetSelection, setAssetSelection} = - useAssetSelectionInput(partiallyFiltered); + const {filterInput, filtered, loading, assetSelection, setAssetSelection} = + useAssetSelectionInput(partiallyFiltered, !assets); - useBlockTraceUntilTrue('useAllAssets', !!assets?.length && !fetchResult.loading); + useBlockTraceUntilTrue('useAllAssets', !!assets?.length && !loading); const {displayPathForAsset, displayed} = useMemo( () => @@ -225,7 +225,7 @@ export const AssetsCatalogTable = ({ const refreshState = useRefreshAtInterval({ refresh: query, - intervalMs: FIFTEEN_SECONDS, + intervalMs: 4 * FIFTEEN_SECONDS, leading: true, }); @@ -255,7 +255,7 @@ export const AssetsCatalogTable = ({ ({})); +jest.mock( + 'lodash/throttle', + () => + (fn: (...args: any[]) => any) => + (...args: any[]) => + fn(...args), +); // These files must be mocked because useVirtualizer tries to create a ResizeObserver, // and the component tree fails to mount. @@ -91,8 +100,10 @@ describe('AssetView', () => { describe('Launch button', () => { it('shows the "Materialize" button for a software-defined asset', async () => { - render(); - expect(await screen.findByText('Materialize')).toBeVisible(); + await act(() => render()); + await waitFor(async () => { + expect(await screen.findByText('Materialize')).toBeVisible(); + }); }); it('shows the "Observe" button for a software-defined source asset', async () => { diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/useAssetDefinitionFilterState.oss.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/useAssetDefinitionFilterState.oss.tsx index b2681d857761b..c90e38ceb3e17 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/useAssetDefinitionFilterState.oss.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/useAssetDefinitionFilterState.oss.tsx @@ -10,7 +10,8 @@ import { DefinitionTag, } from '../graphql/types'; import {useQueryPersistedState} from '../hooks/useQueryPersistedState'; -import {Tag, doesFilterArrayMatchValueArray} from '../ui/Filters/useDefinitionTagFilter'; +import {doesFilterArrayMatchValueArray} from '../ui/Filters/doesFilterArrayMatchValueArray'; +import {Tag} from '../ui/Filters/useDefinitionTagFilter'; import {buildRepoAddress} from '../workspace/buildRepoAddress'; import {RepoAddress} from '../workspace/types'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/automation/MergedAutomationRoot.tsx b/js_modules/dagster-ui/packages/ui-core/src/automation/MergedAutomationRoot.tsx index 78155c7ff3ca3..b5c1ba848d703 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/automation/MergedAutomationRoot.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/automation/MergedAutomationRoot.tsx @@ -25,9 +25,9 @@ import {makeAutomationKey} from '../sensors/makeSensorKey'; import {useFilters} from '../ui/BaseFilters'; import {useStaticSetFilter} from '../ui/BaseFilters/useStaticSetFilter'; import {CheckAllBox} from '../ui/CheckAllBox'; +import {doesFilterArrayMatchValueArray} from '../ui/Filters/doesFilterArrayMatchValueArray'; import {useCodeLocationFilter} from '../ui/Filters/useCodeLocationFilter'; import { - doesFilterArrayMatchValueArray, useDefinitionTagFilterWithManagedState, useTagsForObjects, } from '../ui/Filters/useDefinitionTagFilter'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/graph/asyncGraphLayout.ts b/js_modules/dagster-ui/packages/ui-core/src/graph/asyncGraphLayout.ts index 5ecd65b65199a..f38c20e1d25e2 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/graph/asyncGraphLayout.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/graph/asyncGraphLayout.ts @@ -35,8 +35,6 @@ const asyncGetFullOpLayout = asyncMemoize((ops: ILayoutOp[], opts: LayoutOpGraph }); }, _opLayoutCacheKey); -// Asset Graph - const _assetLayoutCacheKey = (graphData: GraphData, opts: LayoutAssetGraphOptions) => { // Note: The "show secondary edges" toggle means that we need a cache key that incorporates // both the displayed nodes and the displayed edges. @@ -202,7 +200,7 @@ export function useAssetLayout( const graphData = useMemo(() => ({..._graphData, expandedGroups}), [expandedGroups, _graphData]); - const cacheKey = _assetLayoutCacheKey(graphData, opts); + const cacheKey = useMemo(() => _assetLayoutCacheKey(graphData, opts), [graphData, opts]); const nodeCount = Object.keys(graphData.nodes).length; const runAsync = nodeCount >= ASYNC_LAYOUT_SOLID_COUNT; diff --git a/js_modules/dagster-ui/packages/ui-core/src/live-data-provider/LiveDataScheduler.tsx b/js_modules/dagster-ui/packages/ui-core/src/live-data-provider/LiveDataScheduler.tsx new file mode 100644 index 0000000000000..6dc5aa18a6334 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/live-data-provider/LiveDataScheduler.tsx @@ -0,0 +1,36 @@ +import type {LiveDataThread} from './LiveDataThread'; + +/** + * This exists as a separate class to allow Jest to mock it + */ +export class LiveDataScheduler { + private thread: LiveDataThread; + + constructor(thread: LiveDataThread) { + this.thread = thread; + } + private _starting = false; + private _stopping = false; + + public scheduleStartFetchLoop(doStart: () => void) { + if (this._starting) { + return; + } + this._starting = true; + setTimeout(() => { + doStart(); + this._starting = false; + }, 50); + } + + public scheduleStopFetchLoop(doStop: () => void) { + if (this._stopping) { + return; + } + this._stopping = true; + setTimeout(() => { + doStop(); + this._stopping = false; + }, 50); + } +} diff --git a/js_modules/dagster-ui/packages/ui-core/src/live-data-provider/LiveDataThread.tsx b/js_modules/dagster-ui/packages/ui-core/src/live-data-provider/LiveDataThread.tsx index 73534ba1ea17f..cc157974f9fd3 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/live-data-provider/LiveDataThread.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/live-data-provider/LiveDataThread.tsx @@ -1,3 +1,4 @@ +import {LiveDataScheduler} from './LiveDataScheduler'; import {LiveDataThreadManager} from './LiveDataThreadManager'; import {BATCH_PARALLEL_FETCHES, BATCH_SIZE, threadIDToLimits} from './util'; @@ -19,6 +20,8 @@ export class LiveDataThread { return {}; } + private _scheduler: LiveDataScheduler; + constructor( id: string, manager: LiveDataThreadManager, @@ -33,6 +36,7 @@ export class LiveDataThread { this.listenersCount = {}; this.manager = manager; this.intervals = []; + this._scheduler = new LiveDataScheduler(this); } public setPollRate(pollRate: number) { @@ -53,9 +57,7 @@ export class LiveDataThread { if (this.listenersCount[key] === 0) { delete this.listenersCount[key]; } - if (this.getObservedKeys().length === 0) { - this.stopFetchLoop(); - } + this.stopFetchLoop(false); } public getObservedKeys() { @@ -63,19 +65,25 @@ export class LiveDataThread { } public startFetchLoop() { - if (this.activeFetches !== this.parallelFetches) { - requestAnimationFrame(this._batchedQueryKeys); - } - if (this.intervals.length !== this.parallelFetches) { - this.intervals.push(setInterval(this._batchedQueryKeys, 5000)); - } + this._scheduler.scheduleStartFetchLoop(() => { + if (this.activeFetches !== this.parallelFetches) { + requestAnimationFrame(this._batchedQueryKeys); + } + if (this.intervals.length !== this.parallelFetches) { + this.intervals.push(setInterval(this._batchedQueryKeys, 5000)); + } + }); } - public stopFetchLoop() { - this.intervals.forEach((id) => { - clearInterval(id); + public stopFetchLoop(force: boolean) { + this._scheduler.scheduleStopFetchLoop(() => { + if (force || this.getObservedKeys().length === 0) { + this.intervals.forEach((id) => { + clearInterval(id); + }); + this.intervals = []; + } }); - this.intervals = []; } private _batchedQueryKeys = async () => { diff --git a/js_modules/dagster-ui/packages/ui-core/src/live-data-provider/LiveDataThreadManager.tsx b/js_modules/dagster-ui/packages/ui-core/src/live-data-provider/LiveDataThreadManager.tsx index e7118db851ebf..cfc51e7028c91 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/live-data-provider/LiveDataThreadManager.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/live-data-provider/LiveDataThreadManager.tsx @@ -174,7 +174,7 @@ export class LiveDataThreadManager { private pause() { this.isPaused = true; Object.values(this.threads).forEach((thread) => { - thread.stopFetchLoop(); + thread.stopFetchLoop(true); }); } diff --git a/js_modules/dagster-ui/packages/ui-core/src/setupTests.ts b/js_modules/dagster-ui/packages/ui-core/src/setupTests.ts index 1b61dbdb34901..8486deec66a7d 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/setupTests.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/setupTests.ts @@ -99,3 +99,6 @@ class MockBroadcastChannel { } (global as any).BroadcastChannel = MockBroadcastChannel; + +// eslint-disable-next-line @typescript-eslint/no-require-imports +require('fast-text-encoding'); diff --git a/js_modules/dagster-ui/packages/ui-core/src/ui/Filters/doesFilterArrayMatchValueArray.tsx b/js_modules/dagster-ui/packages/ui-core/src/ui/Filters/doesFilterArrayMatchValueArray.tsx new file mode 100644 index 0000000000000..a206aca75ced0 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/ui/Filters/doesFilterArrayMatchValueArray.tsx @@ -0,0 +1,25 @@ +import isEqual from 'lodash/isEqual'; + +// This function checks if every element in `filterArray` has at least one match in `valueArray` based on the provided `isMatch` comparison function. +// - `filterArray`: The array containing elements to be matched. +// - `valueArray`: The array to search for matches. +// - `isMatch`: A custom comparator function (defaults to deep equality using `lodash/isEqual`). +// Returns `false` if `filterArray` is non-empty and `valueArray` is empty (no matches possible). +// Otherwise, checks if all elements in `filterArray` have at least one corresponding match in `valueArray`. +// Uses `Array.prototype.some()` to verify if any `filterArray` element lacks a match and returns `false` in such cases. +export function doesFilterArrayMatchValueArray( + filterArray: T[], + valueArray: V[], + isMatch: (value1: T, value2: V) => boolean = (val1, val2) => { + return isEqual(val1, val2); + }, +) { + if (filterArray.length && !valueArray.length) { + return false; + } + return !filterArray.some( + (filterTag) => + // If no asset tags match this filter tag return true + !valueArray.find((value) => isMatch(filterTag, value)), + ); +} diff --git a/js_modules/dagster-ui/packages/ui-core/src/ui/Filters/useDefinitionTagFilter.tsx b/js_modules/dagster-ui/packages/ui-core/src/ui/Filters/useDefinitionTagFilter.tsx index adeaa9855bf60..b309bbb526dc9 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/ui/Filters/useDefinitionTagFilter.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/ui/Filters/useDefinitionTagFilter.tsx @@ -1,4 +1,3 @@ -import isEqual from 'lodash/isEqual'; import memoize from 'lodash/memoize'; import {useCallback, useMemo} from 'react'; @@ -94,23 +93,6 @@ export function useTagsForObjects( ); } -export function doesFilterArrayMatchValueArray( - filterArray: T[], - valueArray: V[], - isMatch: (value1: T, value2: V) => boolean = (val1, val2) => { - return isEqual(val1, val2); - }, -) { - if (filterArray.length && !valueArray.length) { - return false; - } - return !filterArray.some( - (filterTag) => - // If no asset tags match this filter tag return true - !valueArray.find((value) => isMatch(filterTag, value)), - ); -} - export const BaseConfig: StaticBaseConfig = { name: 'Tag', icon: 'tag', diff --git a/js_modules/dagster-ui/packages/ui-core/src/util/__tests__/generateObjectHash.test.ts b/js_modules/dagster-ui/packages/ui-core/src/util/__tests__/generateObjectHash.test.ts new file mode 100644 index 0000000000000..29120193bdbb2 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/util/__tests__/generateObjectHash.test.ts @@ -0,0 +1,99 @@ +import {generateObjectHashStream} from '../generateObjectHash'; + +describe('generateObjectHashStream', () => { + test('hashes a simple object correctly', async () => { + const obj1 = {b: 2, a: 1}; + const obj2 = {a: 1, b: 2}; + + const hash1 = await generateObjectHashStream(obj1); + const hash2 = await generateObjectHashStream(obj2); + + expect(hash1).toBe(hash2); // Should be equal since keys are sorted + }); + + test('hashes nested objects and arrays correctly', async () => { + const obj1 = { + user: { + id: 1, + name: 'Alice', + roles: ['admin', 'user'], + }, + active: true, + }; + + const obj2 = { + active: true, + user: { + roles: ['admin', 'user'], + name: 'Alice', + id: 1, + }, + }; + + const hash1 = await generateObjectHashStream(obj1); + const hash2 = await generateObjectHashStream(obj2); + + expect(hash1).toBe(hash2); // Should be equal due to sorted keys + }); + + test('differentiates between different objects', async () => { + const obj1 = {a: [1]}; + const obj2 = {a: [2]}; + const hash1 = await generateObjectHashStream(obj1); + const hash2 = await generateObjectHashStream(obj2); + expect(hash1).not.toBe(hash2); // Should be different + }); + + test('handles arrays correctly', async () => { + const arr1 = [1, 2, 3]; + const arr2 = [1, 2, 3]; + const arr3 = [3, 2, 1]; + + const hash1 = await generateObjectHashStream(arr1); + const hash2 = await generateObjectHashStream(arr2); + const hash3 = await generateObjectHashStream(arr3); + + expect(hash1).toBe(hash2); + expect(hash1).not.toBe(hash3); + }); + + test('handles empty objects and arrays', async () => { + const emptyObj = {}; + const emptyArr: any[] = []; + + const hashObj = await generateObjectHashStream(emptyObj); + const hashArr = await generateObjectHashStream(emptyArr); + + expect(hashObj).not.toEqual(hashArr); + }); + + test('handles nested arrays correctly', async () => { + const obj1 = { + a: [ + [1, 2], + [3, 4], + ], + }; + const obj2 = { + a: [ + [1, 2], + [3, 5], + ], + }; + + const hash1 = await generateObjectHashStream(obj1); + const hash2 = await generateObjectHashStream(obj2); + + expect(hash1).not.toBe(hash2); + }); + + test('handles different property types', async () => { + const obj1 = {a: 1, b: 'text', c: true}; + const obj2 = {a: 1, b: 'text', c: false}; + + const hash1 = await generateObjectHashStream(obj1); + const hash2 = await generateObjectHashStream(obj2); + + expect(hash1).not.toBe(hash2); + }); +}); diff --git a/js_modules/dagster-ui/packages/ui-core/src/util/generateObjectHash.ts b/js_modules/dagster-ui/packages/ui-core/src/util/generateObjectHash.ts new file mode 100644 index 0000000000000..a9e3e664cfae1 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/util/generateObjectHash.ts @@ -0,0 +1,103 @@ +import SparkMD5 from 'spark-md5'; + +/** + * Generates a hash for a JSON-serializable object using incremental JSON serialization + * and SparkMD5 for hashing. + * + * @param obj - The JSON-serializable object to hash. + * @param replacer - Optional JSON.stringify replacer function. + * @returns A Promise that resolves to the hexadecimal string representation of the hash. + */ +export function generateObjectHashStream( + obj: any, + replacer?: (key: string, value: any) => any, +): string { + const hash = new SparkMD5.ArrayBuffer(); + const encoder = new TextEncoder(); + + type Frame = { + obj: any; + keys: string[] | number[]; + index: number; + isArray: boolean; + isFirst: boolean; + }; + + const stack: Frame[] = []; + const isRootArray = Array.isArray(obj); + const initialKeys = isRootArray ? Array.from(Array(obj.length).keys()) : Object.keys(obj).sort(); + stack.push({ + obj, + keys: initialKeys, + index: 0, + isArray: isRootArray, + isFirst: true, + }); + + hash.append(encoder.encode(isRootArray ? '[' : '{')); + + while (stack.length > 0) { + const currentFrame = stack[stack.length - 1]!; + + if (currentFrame.index >= currentFrame.keys.length) { + stack.pop(); + hash.append(encoder.encode(currentFrame.isArray ? ']' : '}')); + if (stack.length > 0) { + const parentFrame = stack[stack.length - 1]!; + parentFrame.isFirst = false; + } + continue; + } + + if (!currentFrame.isFirst) { + hash.append(encoder.encode(',')); + } + currentFrame.isFirst = false; + + const key = currentFrame.keys[currentFrame.index]; + currentFrame.index += 1; + + let value: any; + if (currentFrame.isArray) { + value = currentFrame.obj[key as number]; + } else { + value = currentFrame.obj[key as string]; + } + + value = replacer ? replacer(currentFrame.isArray ? '' : String(key), value) : value; + + if (!currentFrame.isArray) { + const serializedKey = JSON.stringify(key) + ':'; + hash.append(encoder.encode(serializedKey)); + } + + if (value && typeof value === 'object') { + if (Array.isArray(value)) { + hash.append(encoder.encode('[')); + const childKeys = Array.from(Array(value.length).keys()); + stack.push({ + obj: value, + keys: childKeys, + index: 0, + isArray: true, + isFirst: true, + }); + } else { + const childKeys = Object.keys(value).sort(); + hash.append(encoder.encode('{')); + stack.push({ + obj: value, + keys: childKeys, + index: 0, + isArray: false, + isFirst: true, + }); + } + } else { + const serializedValue = JSON.stringify(value); + hash.append(encoder.encode(serializedValue)); + } + } + + return hash.end(); +} diff --git a/js_modules/dagster-ui/yarn.lock b/js_modules/dagster-ui/yarn.lock index cc614aadaf3b3..01cfe4d490055 100644 --- a/js_modules/dagster-ui/yarn.lock +++ b/js_modules/dagster-ui/yarn.lock @@ -3727,6 +3727,7 @@ __metadata: "@types/color": "npm:^3.0.2" "@types/dagre": "npm:^0.7.42" "@types/faker": "npm:^5.1.7" + "@types/fast-text-encoding": "npm:^1.0.3" "@types/graphql": "npm:^14.5.0" "@types/jest": "npm:^29.5.11" "@types/lodash": "npm:^4.14.145" @@ -3737,6 +3738,7 @@ __metadata: "@types/react-dom": "npm:^18.3.1" "@types/react-router": "npm:^5.1.17" "@types/react-router-dom": "npm:^5.3.3" + "@types/spark-md5": "npm:^3" "@types/testing-library__jest-dom": "npm:^5.14.2" "@types/ws": "npm:^6.0.3" "@typescript-eslint/eslint-plugin": "npm:^8.9.0" @@ -3768,6 +3770,7 @@ __metadata: eslint-plugin-unused-imports: "npm:^4.1.4" fake-indexeddb: "npm:^4.0.2" faker: "npm:5.5.3" + fast-text-encoding: "npm:^1.0.6" fuse.js: "npm:^6.4.2" graphql: "npm:^16.8.1" graphql-codegen-persisted-query-ids: "npm:^0.2.0" @@ -3801,6 +3804,7 @@ __metadata: remark: "npm:^14.0.2" remark-gfm: "npm:3.0.1" resize-observer-polyfill: "npm:^1.5.1" + spark-md5: "npm:^3.0.2" storybook: "npm:^8.2.7" strip-markdown: "npm:^6.0.0" styled-components: "npm:^6" @@ -7164,6 +7168,13 @@ __metadata: languageName: node linkType: hard +"@types/fast-text-encoding@npm:^1.0.3": + version: 1.0.3 + resolution: "@types/fast-text-encoding@npm:1.0.3" + checksum: 10/34ec2bbaf3e3ee36b7b74375293becc735378f77e9cd93b810ad988b42991ee80d30fb942e6ba03adfc1f0cb0e2024a0aeee84475847563ed6782e21c4c0f5f0 + languageName: node + linkType: hard + "@types/graceful-fs@npm:^4.1.2, @types/graceful-fs@npm:^4.1.3": version: 4.1.6 resolution: "@types/graceful-fs@npm:4.1.6" @@ -7565,6 +7576,13 @@ __metadata: languageName: node linkType: hard +"@types/spark-md5@npm:^3": + version: 3.0.5 + resolution: "@types/spark-md5@npm:3.0.5" + checksum: 10/b543313e8669db34259aa67cff281f63b6746e08711e2b93d653cbb32ec63bb6153e75eeb534d3e874b5a6c1cb8cbe099dd85f9f912b23d9b0f4d51f3e968a2e + languageName: node + linkType: hard + "@types/stack-utils@npm:^2.0.0": version: 2.0.1 resolution: "@types/stack-utils@npm:2.0.1" @@ -13221,6 +13239,13 @@ __metadata: languageName: node linkType: hard +"fast-text-encoding@npm:^1.0.6": + version: 1.0.6 + resolution: "fast-text-encoding@npm:1.0.6" + checksum: 10/f7b9e2e7a21e4ae5f4b8d3729850be83fb45052b28c9c38c09b8366463a291d6dc5448359238bdaf87f6a9e907d5895a94319a2c5e0e9f0786859ad6312d1d06 + languageName: node + linkType: hard + "fast-url-parser@npm:^1.1.3": version: 1.1.3 resolution: "fast-url-parser@npm:1.1.3" @@ -21482,6 +21507,13 @@ __metadata: languageName: node linkType: hard +"spark-md5@npm:^3.0.2": + version: 3.0.2 + resolution: "spark-md5@npm:3.0.2" + checksum: 10/60981e181a296b2d16064ef86607f78d7eb1e08a5f39366239bb9cdd6bc3838fb2f667f2506e81c8d5c71965cdd6f18a17fb1c9a8368eeb407b9dd8188e95473 + languageName: node + linkType: hard + "split-on-first@npm:^1.0.0": version: 1.1.0 resolution: "split-on-first@npm:1.1.0"