diff --git a/js_modules/dagster-ui/packages/ui-core/package.json b/js_modules/dagster-ui/packages/ui-core/package.json index 23d40dd9517ab..a17017e2ce43d 100644 --- a/js_modules/dagster-ui/packages/ui-core/package.json +++ b/js_modules/dagster-ui/packages/ui-core/package.json @@ -53,6 +53,7 @@ "dayjs": "^1.11.7", "deepmerge": "^4.2.2", "fake-indexeddb": "^4.0.2", + "fast-text-encoding": "^1.0.6", "fuse.js": "^6.4.2", "graphql": "^16.8.1", "graphql-codegen-persisted-query-ids": "^0.2.0", @@ -71,6 +72,7 @@ "rehype-sanitize": "^5.0.1", "remark": "^14.0.2", "remark-gfm": "3.0.1", + "spark-md5": "^3.0.2", "strip-markdown": "^6.0.0", "subscriptions-transport-ws": "^0.9.15", "worker-loader": "^3.0.8", @@ -117,6 +119,7 @@ "@types/color": "^3.0.2", "@types/dagre": "^0.7.42", "@types/faker": "^5.1.7", + "@types/fast-text-encoding": "^1.0.3", "@types/graphql": "^14.5.0", "@types/jest": "^29.5.11", "@types/lodash": "^4.14.145", @@ -127,6 +130,7 @@ "@types/react-dom": "^18.3.1", "@types/react-router": "^5.1.17", "@types/react-router-dom": "^5.3.3", + "@types/spark-md5": "^3", "@types/testing-library__jest-dom": "^5.14.2", "@types/ws": "^6.0.3", "@typescript-eslint/eslint-plugin": "^8.9.0", diff --git a/js_modules/dagster-ui/packages/ui-core/src/app/Util.tsx b/js_modules/dagster-ui/packages/ui-core/src/app/Util.tsx index 649cab2aa8af8..f09f482ea1555 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/app/Util.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/app/Util.tsx @@ -160,6 +160,8 @@ export function indexedDBAsyncMemoize> = {}; + async function genHashKey(arg: T, ...rest: any[]) { const hash = hashFn ? hashFn(arg, ...rest) : arg; @@ -182,17 +184,21 @@ export function indexedDBAsyncMemoize { + const result = await fn(arg, ...rest); + // Resolve the promise before storing the result in IndexedDB + res(result); + if (lru) { + await lru.set(hashKey, result, { + // Some day in the year 2050... + expiry: new Date(9 ** 13), + }); + delete hashToPromise[hashKey]; + } }); } + resolve(await hashToPromise[hashKey]!); }); }) as any; ret.isCached = async (arg: T, ...rest: any) => { diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx index 51dd6749fa8fe..76643fac97872 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx @@ -98,11 +98,11 @@ type Props = { export const MINIMAL_SCALE = 0.6; export const GROUPS_ONLY_SCALE = 0.15; +const DEFAULT_SET_HIDE_NODES_MATCH = (_node: AssetNodeForGraphQueryFragment) => true; + export const AssetGraphExplorer = (props: Props) => { const fullAssetGraphData = useFullAssetGraphData(props.fetchOptions); - const [hideNodesMatching, setHideNodesMatching] = useState( - () => (_node: AssetNodeForGraphQueryFragment) => true, - ); + const [hideNodesMatching, setHideNodesMatching] = useState(() => DEFAULT_SET_HIDE_NODES_MATCH); const { loading: graphDataLoading, @@ -110,7 +110,13 @@ export const AssetGraphExplorer = (props: Props) => { assetGraphData, graphQueryItems, allAssetKeys, - } = useAssetGraphData(props.explorerPath.opsQuery, {...props.fetchOptions, hideNodesMatching}); + } = useAssetGraphData( + props.explorerPath.opsQuery, + useMemo( + () => ({...props.fetchOptions, hideNodesMatching}), + [props.fetchOptions, hideNodesMatching], + ), + ); const {explorerPath, onChangeExplorerPath} = props; diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/useAssetGraphData.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/useAssetGraphData.tsx index 431ca45c704ed..ff9b2f98f68da 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/useAssetGraphData.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/useAssetGraphData.tsx @@ -1,3 +1,4 @@ +import {memoize} from 'lodash'; import keyBy from 'lodash/keyBy'; import reject from 'lodash/reject'; import throttle from 'lodash/throttle'; @@ -18,7 +19,7 @@ import { } from './types/useAssetGraphData.types'; import {usePrefixedCacheKey} from '../app/AppProvider'; import {GraphQueryItem} from '../app/GraphQueryImpl'; -import {asyncMemoize} from '../app/Util'; +import {indexedDBAsyncMemoize} from '../app/Util'; import {AssetKey} from '../assets/types'; import {AssetGroupSelector, PipelineSelector} from '../graphql/types'; import {useIndexedDBCachedQuery} from '../search/useIndexedDBCachedQuery'; @@ -29,7 +30,10 @@ export interface AssetGraphFetchScope { pipelineSelector?: PipelineSelector; groupSelector?: AssetGroupSelector; kinds?: string[]; - loading: boolean; // true if we shouldn't start handling any input. + + // This is used to indicate we shouldn't start handling any input. + // This is used by pages where `hideNodesMatching` is only available asynchronously. + loading?: boolean; } export type AssetGraphQueryItem = GraphQueryItem & { @@ -121,7 +125,6 @@ export function useAssetGraphData(opsQuery: string, options: AssetGraphFetchScop ); const [state, setState] = useState(INITIAL_STATE); - const {assetGraphData, graphAssetKeys, allAssetKeys} = state; const {kinds, hideEdgesToNodesOutsideQuery} = options; @@ -145,6 +148,7 @@ export function useAssetGraphData(opsQuery: string, options: AssetGraphFetchScop flagAssetSelectionSyntax: featureEnabled(FeatureFlag.flagAssetSelectionSyntax), })?.then((data) => { if (lastProcessedRequestRef.current < requestId) { + lastProcessedRequestRef.current = requestId; setState(data); if (requestId === currentRequestRef.current) { setGraphDataLoading(false); @@ -164,10 +168,10 @@ export function useAssetGraphData(opsQuery: string, options: AssetGraphFetchScop return { loading, fetchResult, - assetGraphData, + assetGraphData: state.assetGraphData, graphQueryItems, - graphAssetKeys, - allAssetKeys, + graphAssetKeys: state.graphAssetKeys, + allAssetKeys: state.allAssetKeys, }; } @@ -294,22 +298,27 @@ export const ASSET_GRAPH_QUERY = gql` `; const computeGraphData = throttle( - asyncMemoize( - computeGraphDataWrapper, - ), + indexedDBAsyncMemoize< + ComputeGraphDataMessageType, + GraphDataState, + typeof computeGraphDataWrapper + >(computeGraphDataWrapper, (props) => { + return JSON.stringify(props); + }), 2000, {leading: true}, ); +const getWorker = memoize(() => new Worker(new URL('./ComputeGraphData.worker', import.meta.url))); + async function computeGraphDataWrapper( props: Omit, ): Promise { if (featureEnabled(FeatureFlag.flagAssetSelectionWorker)) { + const worker = getWorker(); return new Promise((resolve) => { - const worker = new Worker(new URL('./ComputeGraphData.worker', import.meta.url)); worker.addEventListener('message', (event) => { resolve(event.data as GraphDataState); - worker.terminate(); }); const message: ComputeGraphDataMessageType = { type: 'computeGraphData', diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-selection/useAssetSelectionFiltering.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-selection/useAssetSelectionFiltering.tsx index e6019ef2b2ee1..564240d2c8eeb 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-selection/useAssetSelectionFiltering.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-selection/useAssetSelectionFiltering.tsx @@ -27,6 +27,7 @@ export const useAssetSelectionFiltering = < [assets], ); + const assetsByKeyStringified = useMemo(() => JSON.stringify(assetsByKey), [assetsByKey]); const {loading, graphQueryItems, graphAssetKeys} = useAssetGraphData( assetSelection, useMemo( @@ -37,17 +38,24 @@ export const useAssetSelectionFiltering = < }, loading: !!assetsLoading, }), - [assetsByKey, assetsLoading], + // eslint-disable-next-line react-hooks/exhaustive-deps + [assetsByKeyStringified, assetsLoading], ), ); const filtered = useMemo(() => { + if (!assetSelection) { + return assets; + } return ( graphAssetKeys - .map((key) => assetsByKey[tokenForAssetKey(key)]!) + .map((key) => { + return assetsByKey[tokenForAssetKey(key)]!; + }) + .filter((a) => a) .sort((a, b) => COMMON_COLLATOR.compare(a.key.path.join(''), b.key.path.join(''))) ?? [] ); - }, [graphAssetKeys, assetsByKey]); + }, [assetSelection, graphAssetKeys, assets, assetsByKey]); const filteredByKey = useMemo( () => Object.fromEntries(filtered.map((asset) => [tokenForAssetKey(asset.key), asset])), diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetGroupRoot.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetGroupRoot.tsx index 2e4a8fe8ccfc4..51c1544fbc3df 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetGroupRoot.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetGroupRoot.tsx @@ -93,7 +93,7 @@ export const AssetGroupRoot = ({ [history, openInNewTab], ); - const fetchOptions = React.useMemo(() => ({groupSelector}), [groupSelector]); + const fetchOptions = React.useMemo(() => ({groupSelector, loading: false}), [groupSelector]); const lineageOptions = React.useMemo( () => ({preferAssetRendering: true, explodeComposites: true}), diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetsCatalogTable.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetsCatalogTable.tsx index 55c4214447efa..5265eaae1d4d6 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetsCatalogTable.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetsCatalogTable.tsx @@ -211,7 +211,7 @@ export const AssetsCatalogTable = ({ kindFilter, } = useAssetCatalogFiltering({assets}); const {filterInput, filtered, loading, assetSelection, setAssetSelection} = - useAssetSelectionInput(partiallyFiltered, !!assets); + useAssetSelectionInput(partiallyFiltered, !assets); useBlockTraceUntilTrue('useAllAssets', !!assets?.length && !loading); diff --git a/js_modules/dagster-ui/packages/ui-core/src/graph/asyncGraphLayout.ts b/js_modules/dagster-ui/packages/ui-core/src/graph/asyncGraphLayout.ts index 5ecd65b65199a..f38c20e1d25e2 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/graph/asyncGraphLayout.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/graph/asyncGraphLayout.ts @@ -35,8 +35,6 @@ const asyncGetFullOpLayout = asyncMemoize((ops: ILayoutOp[], opts: LayoutOpGraph }); }, _opLayoutCacheKey); -// Asset Graph - const _assetLayoutCacheKey = (graphData: GraphData, opts: LayoutAssetGraphOptions) => { // Note: The "show secondary edges" toggle means that we need a cache key that incorporates // both the displayed nodes and the displayed edges. @@ -202,7 +200,7 @@ export function useAssetLayout( const graphData = useMemo(() => ({..._graphData, expandedGroups}), [expandedGroups, _graphData]); - const cacheKey = _assetLayoutCacheKey(graphData, opts); + const cacheKey = useMemo(() => _assetLayoutCacheKey(graphData, opts), [graphData, opts]); const nodeCount = Object.keys(graphData.nodes).length; const runAsync = nodeCount >= ASYNC_LAYOUT_SOLID_COUNT; diff --git a/js_modules/dagster-ui/packages/ui-core/src/setupTests.ts b/js_modules/dagster-ui/packages/ui-core/src/setupTests.ts index 1b61dbdb34901..8486deec66a7d 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/setupTests.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/setupTests.ts @@ -99,3 +99,6 @@ class MockBroadcastChannel { } (global as any).BroadcastChannel = MockBroadcastChannel; + +// eslint-disable-next-line @typescript-eslint/no-require-imports +require('fast-text-encoding'); diff --git a/js_modules/dagster-ui/packages/ui-core/src/util/__tests__/generateObjectHash.test.ts b/js_modules/dagster-ui/packages/ui-core/src/util/__tests__/generateObjectHash.test.ts new file mode 100644 index 0000000000000..29120193bdbb2 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/util/__tests__/generateObjectHash.test.ts @@ -0,0 +1,99 @@ +import {generateObjectHashStream} from '../generateObjectHash'; + +describe('generateObjectHashStream', () => { + test('hashes a simple object correctly', async () => { + const obj1 = {b: 2, a: 1}; + const obj2 = {a: 1, b: 2}; + + const hash1 = await generateObjectHashStream(obj1); + const hash2 = await generateObjectHashStream(obj2); + + expect(hash1).toBe(hash2); // Should be equal since keys are sorted + }); + + test('hashes nested objects and arrays correctly', async () => { + const obj1 = { + user: { + id: 1, + name: 'Alice', + roles: ['admin', 'user'], + }, + active: true, + }; + + const obj2 = { + active: true, + user: { + roles: ['admin', 'user'], + name: 'Alice', + id: 1, + }, + }; + + const hash1 = await generateObjectHashStream(obj1); + const hash2 = await generateObjectHashStream(obj2); + + expect(hash1).toBe(hash2); // Should be equal due to sorted keys + }); + + test('differentiates between different objects', async () => { + const obj1 = {a: [1]}; + const obj2 = {a: [2]}; + const hash1 = await generateObjectHashStream(obj1); + const hash2 = await generateObjectHashStream(obj2); + expect(hash1).not.toBe(hash2); // Should be different + }); + + test('handles arrays correctly', async () => { + const arr1 = [1, 2, 3]; + const arr2 = [1, 2, 3]; + const arr3 = [3, 2, 1]; + + const hash1 = await generateObjectHashStream(arr1); + const hash2 = await generateObjectHashStream(arr2); + const hash3 = await generateObjectHashStream(arr3); + + expect(hash1).toBe(hash2); + expect(hash1).not.toBe(hash3); + }); + + test('handles empty objects and arrays', async () => { + const emptyObj = {}; + const emptyArr: any[] = []; + + const hashObj = await generateObjectHashStream(emptyObj); + const hashArr = await generateObjectHashStream(emptyArr); + + expect(hashObj).not.toEqual(hashArr); + }); + + test('handles nested arrays correctly', async () => { + const obj1 = { + a: [ + [1, 2], + [3, 4], + ], + }; + const obj2 = { + a: [ + [1, 2], + [3, 5], + ], + }; + + const hash1 = await generateObjectHashStream(obj1); + const hash2 = await generateObjectHashStream(obj2); + + expect(hash1).not.toBe(hash2); + }); + + test('handles different property types', async () => { + const obj1 = {a: 1, b: 'text', c: true}; + const obj2 = {a: 1, b: 'text', c: false}; + + const hash1 = await generateObjectHashStream(obj1); + const hash2 = await generateObjectHashStream(obj2); + + expect(hash1).not.toBe(hash2); + }); +}); diff --git a/js_modules/dagster-ui/packages/ui-core/src/util/generateObjectHash.ts b/js_modules/dagster-ui/packages/ui-core/src/util/generateObjectHash.ts new file mode 100644 index 0000000000000..8f3aedf3ad466 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/util/generateObjectHash.ts @@ -0,0 +1,105 @@ +import SparkMD5 from 'spark-md5'; + +/** + * Generates a hash for a JSON-serializable object using incremental JSON serialization + * and SparkMD5 for hashing. + * + * @param obj - The JSON-serializable object to hash. + * @param replacer - Optional JSON.stringify replacer function. + * @returns A Promise that resolves to the hexadecimal string representation of the hash. + */ +export function generateObjectHashStream( + obj: any, + replacer?: (key: string, value: any) => any, +): string { + console.time('generateObjectHash'); + console.log('generateObjectHash', obj); + const hash = new SparkMD5.ArrayBuffer(); + const encoder = new TextEncoder(); + + type Frame = { + obj: any; + keys: string[] | number[]; + index: number; + isArray: boolean; + isFirst: boolean; + }; + + const stack: Frame[] = []; + const isRootArray = Array.isArray(obj); + const initialKeys = isRootArray ? Array.from(Array(obj.length).keys()) : Object.keys(obj).sort(); + stack.push({ + obj, + keys: initialKeys, + index: 0, + isArray: isRootArray, + isFirst: true, + }); + + hash.append(encoder.encode(isRootArray ? '[' : '{')); + + while (stack.length > 0) { + const currentFrame = stack[stack.length - 1]; + + if (currentFrame.index >= currentFrame.keys.length) { + stack.pop(); + hash.append(encoder.encode(currentFrame.isArray ? ']' : '}')); + if (stack.length > 0) { + const parentFrame = stack[stack.length - 1]; + parentFrame.isFirst = false; + } + continue; + } + + if (!currentFrame.isFirst) { + hash.append(encoder.encode(',')); + } + currentFrame.isFirst = false; + + const key = currentFrame.keys[currentFrame.index]; + currentFrame.index += 1; + + let value: any; + if (currentFrame.isArray) { + value = currentFrame.obj[key as number]; + } else { + value = currentFrame.obj[key as string]; + } + + value = replacer ? replacer(currentFrame.isArray ? '' : String(key), value) : value; + + if (!currentFrame.isArray) { + const serializedKey = JSON.stringify(key) + ':'; + hash.append(encoder.encode(serializedKey)); + } + + if (value && typeof value === 'object') { + if (Array.isArray(value)) { + hash.append(encoder.encode('[')); + const childKeys = Array.from(Array(value.length).keys()); + stack.push({ + obj: value, + keys: childKeys, + index: 0, + isArray: true, + isFirst: true, + }); + } else { + const childKeys = Object.keys(value).sort(); + hash.append(encoder.encode('{')); + stack.push({ + obj: value, + keys: childKeys, + index: 0, + isArray: false, + isFirst: true, + }); + } + } else { + const serializedValue = JSON.stringify(value); + hash.append(encoder.encode(serializedValue)); + } + } + + return hash.end(); +} diff --git a/js_modules/dagster-ui/yarn.lock b/js_modules/dagster-ui/yarn.lock index cc614aadaf3b3..01cfe4d490055 100644 --- a/js_modules/dagster-ui/yarn.lock +++ b/js_modules/dagster-ui/yarn.lock @@ -3727,6 +3727,7 @@ __metadata: "@types/color": "npm:^3.0.2" "@types/dagre": "npm:^0.7.42" "@types/faker": "npm:^5.1.7" + "@types/fast-text-encoding": "npm:^1.0.3" "@types/graphql": "npm:^14.5.0" "@types/jest": "npm:^29.5.11" "@types/lodash": "npm:^4.14.145" @@ -3737,6 +3738,7 @@ __metadata: "@types/react-dom": "npm:^18.3.1" "@types/react-router": "npm:^5.1.17" "@types/react-router-dom": "npm:^5.3.3" + "@types/spark-md5": "npm:^3" "@types/testing-library__jest-dom": "npm:^5.14.2" "@types/ws": "npm:^6.0.3" "@typescript-eslint/eslint-plugin": "npm:^8.9.0" @@ -3768,6 +3770,7 @@ __metadata: eslint-plugin-unused-imports: "npm:^4.1.4" fake-indexeddb: "npm:^4.0.2" faker: "npm:5.5.3" + fast-text-encoding: "npm:^1.0.6" fuse.js: "npm:^6.4.2" graphql: "npm:^16.8.1" graphql-codegen-persisted-query-ids: "npm:^0.2.0" @@ -3801,6 +3804,7 @@ __metadata: remark: "npm:^14.0.2" remark-gfm: "npm:3.0.1" resize-observer-polyfill: "npm:^1.5.1" + spark-md5: "npm:^3.0.2" storybook: "npm:^8.2.7" strip-markdown: "npm:^6.0.0" styled-components: "npm:^6" @@ -7164,6 +7168,13 @@ __metadata: languageName: node linkType: hard +"@types/fast-text-encoding@npm:^1.0.3": + version: 1.0.3 + resolution: "@types/fast-text-encoding@npm:1.0.3" + checksum: 10/34ec2bbaf3e3ee36b7b74375293becc735378f77e9cd93b810ad988b42991ee80d30fb942e6ba03adfc1f0cb0e2024a0aeee84475847563ed6782e21c4c0f5f0 + languageName: node + linkType: hard + "@types/graceful-fs@npm:^4.1.2, @types/graceful-fs@npm:^4.1.3": version: 4.1.6 resolution: "@types/graceful-fs@npm:4.1.6" @@ -7565,6 +7576,13 @@ __metadata: languageName: node linkType: hard +"@types/spark-md5@npm:^3": + version: 3.0.5 + resolution: "@types/spark-md5@npm:3.0.5" + checksum: 10/b543313e8669db34259aa67cff281f63b6746e08711e2b93d653cbb32ec63bb6153e75eeb534d3e874b5a6c1cb8cbe099dd85f9f912b23d9b0f4d51f3e968a2e + languageName: node + linkType: hard + "@types/stack-utils@npm:^2.0.0": version: 2.0.1 resolution: "@types/stack-utils@npm:2.0.1" @@ -13221,6 +13239,13 @@ __metadata: languageName: node linkType: hard +"fast-text-encoding@npm:^1.0.6": + version: 1.0.6 + resolution: "fast-text-encoding@npm:1.0.6" + checksum: 10/f7b9e2e7a21e4ae5f4b8d3729850be83fb45052b28c9c38c09b8366463a291d6dc5448359238bdaf87f6a9e907d5895a94319a2c5e0e9f0786859ad6312d1d06 + languageName: node + linkType: hard + "fast-url-parser@npm:^1.1.3": version: 1.1.3 resolution: "fast-url-parser@npm:1.1.3" @@ -21482,6 +21507,13 @@ __metadata: languageName: node linkType: hard +"spark-md5@npm:^3.0.2": + version: 3.0.2 + resolution: "spark-md5@npm:3.0.2" + checksum: 10/60981e181a296b2d16064ef86607f78d7eb1e08a5f39366239bb9cdd6bc3838fb2f667f2506e81c8d5c71965cdd6f18a17fb1c9a8368eeb407b9dd8188e95473 + languageName: node + linkType: hard + "split-on-first@npm:^1.0.0": version: 1.1.0 resolution: "split-on-first@npm:1.1.0"