mo

js_modules/dagster-ui/packages/ui-core/package.json

@@ -53,6 +53,7 @@
     "dayjs": "^1.11.7",
     "deepmerge": "^4.2.2",
     "fake-indexeddb": "^4.0.2",
+    "fast-text-encoding": "^1.0.6",
     "fuse.js": "^6.4.2",
     "graphql": "^16.8.1",
     "graphql-codegen-persisted-query-ids": "^0.2.0",
@@ -71,6 +72,7 @@
     "rehype-sanitize": "^5.0.1",
     "remark": "^14.0.2",
     "remark-gfm": "3.0.1",
+    "spark-md5": "^3.0.2",
     "strip-markdown": "^6.0.0",
     "subscriptions-transport-ws": "^0.9.15",
     "worker-loader": "^3.0.8",
@@ -117,6 +119,7 @@
     "@types/color": "^3.0.2",
     "@types/dagre": "^0.7.42",
     "@types/faker": "^5.1.7",
+    "@types/fast-text-encoding": "^1.0.3",
     "@types/graphql": "^14.5.0",
     "@types/jest": "^29.5.11",
     "@types/lodash": "^4.14.145",
@@ -127,6 +130,7 @@
     "@types/react-dom": "^18.3.1",
     "@types/react-router": "^5.1.17",
     "@types/react-router-dom": "^5.3.3",
+    "@types/spark-md5": "^3",
     "@types/testing-library__jest-dom": "^5.14.2",
     "@types/ws": "^6.0.3",
     "@typescript-eslint/eslint-plugin": "^8.9.0",

js_modules/dagster-ui/packages/ui-core/src/app/Util.tsx

@@ -160,6 +160,8 @@ export function indexedDBAsyncMemoize<T, R, U extends (arg: T, ...rest: any[]) =
     });
   } catch {}
 
+  const hashToPromise: Record<string, Promise<R>> = {};
+
   async function genHashKey(arg: T, ...rest: any[]) {
     const hash = hashFn ? hashFn(arg, ...rest) : arg;
 
@@ -182,17 +184,21 @@ export function indexedDBAsyncMemoize<T, R, U extends (arg: T, ...rest: any[]) =
         const {value} = await lru.get(hashKey);
         resolve(value);
         return;
-      }
-
-      const result = await fn(arg, ...rest);
-      // Resolve the promise before storing the result in IndexedDB
-      resolve(result);
-      if (lru) {
-        await lru.set(hashKey, result, {
-          // Some day in the year 2050...
-          expiry: new Date(9 ** 13),
+      } else if (!hashToPromise[hashKey]) {
+        hashToPromise[hashKey] = new Promise(async (res) => {
+          const result = await fn(arg, ...rest);
+          // Resolve the promise before storing the result in IndexedDB
+          res(result);
+          if (lru) {
+            await lru.set(hashKey, result, {
+              // Some day in the year 2050...
+              expiry: new Date(9 ** 13),
+            });
+            delete hashToPromise[hashKey];
+          }
         });
       }
+      resolve(await hashToPromise[hashKey]!);
     });
   }) as any;
   ret.isCached = async (arg: T, ...rest: any) => {

js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx

@@ -98,19 +98,25 @@ type Props = {
 export const MINIMAL_SCALE = 0.6;
 export const GROUPS_ONLY_SCALE = 0.15;
 
+const DEFAULT_SET_HIDE_NODES_MATCH = (_node: AssetNodeForGraphQueryFragment) => true;
+
 export const AssetGraphExplorer = (props: Props) => {
   const fullAssetGraphData = useFullAssetGraphData(props.fetchOptions);
-  const [hideNodesMatching, setHideNodesMatching] = useState(
-    () => (_node: AssetNodeForGraphQueryFragment) => true,
-  );
+  const [hideNodesMatching, setHideNodesMatching] = useState(() => DEFAULT_SET_HIDE_NODES_MATCH);
 
   const {
     loading: graphDataLoading,
     fetchResult,
     assetGraphData,
     graphQueryItems,
     allAssetKeys,
-  } = useAssetGraphData(props.explorerPath.opsQuery, {...props.fetchOptions, hideNodesMatching});
+  } = useAssetGraphData(
+    props.explorerPath.opsQuery,
+    useMemo(
+      () => ({...props.fetchOptions, hideNodesMatching}),
+      [props.fetchOptions, hideNodesMatching],
+    ),
+  );
 
   const {explorerPath, onChangeExplorerPath} = props;
 

js_modules/dagster-ui/packages/ui-core/src/asset-graph/useAssetGraphData.tsx

@@ -1,3 +1,4 @@
+import {memoize} from 'lodash';
 import keyBy from 'lodash/keyBy';
 import reject from 'lodash/reject';
 import throttle from 'lodash/throttle';
@@ -18,7 +19,7 @@ import {
 } from './types/useAssetGraphData.types';
 import {usePrefixedCacheKey} from '../app/AppProvider';
 import {GraphQueryItem} from '../app/GraphQueryImpl';
-import {asyncMemoize} from '../app/Util';
+import {indexedDBAsyncMemoize} from '../app/Util';
 import {AssetKey} from '../assets/types';
 import {AssetGroupSelector, PipelineSelector} from '../graphql/types';
 import {useIndexedDBCachedQuery} from '../search/useIndexedDBCachedQuery';
@@ -29,7 +30,10 @@ export interface AssetGraphFetchScope {
   pipelineSelector?: PipelineSelector;
   groupSelector?: AssetGroupSelector;
   kinds?: string[];
-  loading: boolean; // true if we shouldn't start handling any input.
+
+  // This is used to indicate we shouldn't start handling any input.
+  // This is used by pages where `hideNodesMatching` is only available asynchronously.
+  loading?: boolean;
 }
 
 export type AssetGraphQueryItem = GraphQueryItem & {
@@ -121,7 +125,6 @@ export function useAssetGraphData(opsQuery: string, options: AssetGraphFetchScop
   );
 
   const [state, setState] = useState<GraphDataState>(INITIAL_STATE);
-  const {assetGraphData, graphAssetKeys, allAssetKeys} = state;
 
   const {kinds, hideEdgesToNodesOutsideQuery} = options;
 
@@ -145,6 +148,7 @@ export function useAssetGraphData(opsQuery: string, options: AssetGraphFetchScop
       flagAssetSelectionSyntax: featureEnabled(FeatureFlag.flagAssetSelectionSyntax),
     })?.then((data) => {
       if (lastProcessedRequestRef.current < requestId) {
+        lastProcessedRequestRef.current = requestId;
         setState(data);
         if (requestId === currentRequestRef.current) {
           setGraphDataLoading(false);
@@ -164,10 +168,10 @@ export function useAssetGraphData(opsQuery: string, options: AssetGraphFetchScop
   return {
     loading,
     fetchResult,
-    assetGraphData,
+    assetGraphData: state.assetGraphData,
     graphQueryItems,
-    graphAssetKeys,
-    allAssetKeys,
+    graphAssetKeys: state.graphAssetKeys,
+    allAssetKeys: state.allAssetKeys,
   };
 }
 
@@ -294,22 +298,27 @@ export const ASSET_GRAPH_QUERY = gql`
 `;
 
 const computeGraphData = throttle(
-  asyncMemoize<ComputeGraphDataMessageType, GraphDataState, typeof computeGraphDataWrapper>(
-    computeGraphDataWrapper,
-  ),
+  indexedDBAsyncMemoize<
+    ComputeGraphDataMessageType,
+    GraphDataState,
+    typeof computeGraphDataWrapper
+  >(computeGraphDataWrapper, (props) => {
+    return JSON.stringify(props);
+  }),
   2000,
   {leading: true},
 );
 
+const getWorker = memoize(() => new Worker(new URL('./ComputeGraphData.worker', import.meta.url)));
+
 async function computeGraphDataWrapper(
   props: Omit<ComputeGraphDataMessageType, 'type'>,
 ): Promise<GraphDataState> {
   if (featureEnabled(FeatureFlag.flagAssetSelectionWorker)) {
+    const worker = getWorker();
     return new Promise<GraphDataState>((resolve) => {
-      const worker = new Worker(new URL('./ComputeGraphData.worker', import.meta.url));
       worker.addEventListener('message', (event) => {
         resolve(event.data as GraphDataState);
-        worker.terminate();
       });
       const message: ComputeGraphDataMessageType = {
         type: 'computeGraphData',

js_modules/dagster-ui/packages/ui-core/src/asset-selection/useAssetSelectionFiltering.tsx

@@ -27,6 +27,7 @@ export const useAssetSelectionFiltering = <
     [assets],
   );
 
+  const assetsByKeyStringified = useMemo(() => JSON.stringify(assetsByKey), [assetsByKey]);
   const {loading, graphQueryItems, graphAssetKeys} = useAssetGraphData(
     assetSelection,
     useMemo(
@@ -37,17 +38,24 @@ export const useAssetSelectionFiltering = <
         },
         loading: !!assetsLoading,
       }),
-      [assetsByKey, assetsLoading],
+      // eslint-disable-next-line react-hooks/exhaustive-deps
+      [assetsByKeyStringified, assetsLoading],
     ),
   );
 
   const filtered = useMemo(() => {
+    if (!assetSelection) {
+      return assets;
+    }
     return (
       graphAssetKeys
-        .map((key) => assetsByKey[tokenForAssetKey(key)]!)
+        .map((key) => {
+          return assetsByKey[tokenForAssetKey(key)]!;
+        })
+        .filter((a) => a)
         .sort((a, b) => COMMON_COLLATOR.compare(a.key.path.join(''), b.key.path.join(''))) ?? []
     );
-  }, [graphAssetKeys, assetsByKey]);
+  }, [assetSelection, graphAssetKeys, assets, assetsByKey]);
 
   const filteredByKey = useMemo(
     () => Object.fromEntries(filtered.map((asset) => [tokenForAssetKey(asset.key), asset])),

js_modules/dagster-ui/packages/ui-core/src/assets/AssetGroupRoot.tsx

@@ -93,7 +93,7 @@ export const AssetGroupRoot = ({
     [history, openInNewTab],
   );
 
-  const fetchOptions = React.useMemo(() => ({groupSelector}), [groupSelector]);
+  const fetchOptions = React.useMemo(() => ({groupSelector, loading: false}), [groupSelector]);
 
   const lineageOptions = React.useMemo(
     () => ({preferAssetRendering: true, explodeComposites: true}),

js_modules/dagster-ui/packages/ui-core/src/assets/AssetsCatalogTable.tsx

@@ -211,7 +211,7 @@ export const AssetsCatalogTable = ({
     kindFilter,
   } = useAssetCatalogFiltering({assets});
   const {filterInput, filtered, loading, assetSelection, setAssetSelection} =
-    useAssetSelectionInput(partiallyFiltered, !!assets);
+    useAssetSelectionInput(partiallyFiltered, !assets);
 
   useBlockTraceUntilTrue('useAllAssets', !!assets?.length && !loading);
 

js_modules/dagster-ui/packages/ui-core/src/graph/asyncGraphLayout.ts

@@ -35,8 +35,6 @@ const asyncGetFullOpLayout = asyncMemoize((ops: ILayoutOp[], opts: LayoutOpGraph
   });
 }, _opLayoutCacheKey);
 
-// Asset Graph
-
 const _assetLayoutCacheKey = (graphData: GraphData, opts: LayoutAssetGraphOptions) => {
   // Note: The "show secondary edges" toggle means that we need a cache key that incorporates
   // both the displayed nodes and the displayed edges.
@@ -202,7 +200,7 @@ export function useAssetLayout(
 
   const graphData = useMemo(() => ({..._graphData, expandedGroups}), [expandedGroups, _graphData]);
 
-  const cacheKey = _assetLayoutCacheKey(graphData, opts);
+  const cacheKey = useMemo(() => _assetLayoutCacheKey(graphData, opts), [graphData, opts]);
   const nodeCount = Object.keys(graphData.nodes).length;
   const runAsync = nodeCount >= ASYNC_LAYOUT_SOLID_COUNT;
 

js_modules/dagster-ui/packages/ui-core/src/setupTests.ts

@@ -99,3 +99,6 @@ class MockBroadcastChannel {
 }
 
 (global as any).BroadcastChannel = MockBroadcastChannel;
+
+// eslint-disable-next-line @typescript-eslint/no-require-imports
+require('fast-text-encoding');

js_modules/dagster-ui/packages/ui-core/src/util/__tests__/generateObjectHash.test.ts

@@ -0,0 +1,99 @@
+import {generateObjectHashStream} from '../generateObjectHash';
+
+describe('generateObjectHashStream', () => {
+  test('hashes a simple object correctly', async () => {
+    const obj1 = {b: 2, a: 1};
+    const obj2 = {a: 1, b: 2};
+
+    const hash1 = await generateObjectHashStream(obj1);
+    const hash2 = await generateObjectHashStream(obj2);
+
+    expect(hash1).toBe(hash2); // Should be equal since keys are sorted
+  });
+
+  test('hashes nested objects and arrays correctly', async () => {
+    const obj1 = {
+      user: {
+        id: 1,
+        name: 'Alice',
+        roles: ['admin', 'user'],
+      },
+      active: true,
+    };
+
+    const obj2 = {
+      active: true,
+      user: {
+        roles: ['admin', 'user'],
+        name: 'Alice',
+        id: 1,
+      },
+    };
+
+    const hash1 = await generateObjectHashStream(obj1);
+    const hash2 = await generateObjectHashStream(obj2);
+
+    expect(hash1).toBe(hash2); // Should be equal due to sorted keys
+  });
+
+  test('differentiates between different objects', async () => {
+    const obj1 = {a: [1]};
+    const obj2 = {a: [2]};
+    const hash1 = await generateObjectHashStream(obj1);
+    const hash2 = await generateObjectHashStream(obj2);
+    expect(hash1).not.toBe(hash2); // Should be different
+  });
+
+  test('handles arrays correctly', async () => {
+    const arr1 = [1, 2, 3];
+    const arr2 = [1, 2, 3];
+    const arr3 = [3, 2, 1];
+
+    const hash1 = await generateObjectHashStream(arr1);
+    const hash2 = await generateObjectHashStream(arr2);
+    const hash3 = await generateObjectHashStream(arr3);
+
+    expect(hash1).toBe(hash2);
+    expect(hash1).not.toBe(hash3);
+  });
+
+  test('handles empty objects and arrays', async () => {
+    const emptyObj = {};
+    const emptyArr: any[] = [];
+
+    const hashObj = await generateObjectHashStream(emptyObj);
+    const hashArr = await generateObjectHashStream(emptyArr);
+
+    expect(hashObj).not.toEqual(hashArr);
+  });
+
+  test('handles nested arrays correctly', async () => {
+    const obj1 = {
+      a: [
+        [1, 2],
+        [3, 4],
+      ],
+    };
+    const obj2 = {
+      a: [
+        [1, 2],
+        [3, 5],
+      ],
+    };
+
+    const hash1 = await generateObjectHashStream(obj1);
+    const hash2 = await generateObjectHashStream(obj2);
+
+    expect(hash1).not.toBe(hash2);
+  });
+
+  test('handles different property types', async () => {
+    const obj1 = {a: 1, b: 'text', c: true};
+    const obj2 = {a: 1, b: 'text', c: false};
+
+    const hash1 = await generateObjectHashStream(obj1);
+    const hash2 = await generateObjectHashStream(obj2);
+
+    expect(hash1).not.toBe(hash2);
+  });
+});