-
Notifications
You must be signed in to change notification settings - Fork 1.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
move asset selection graph building to webworker
- Loading branch information
Showing
17 changed files
with
272 additions
and
115 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
79 changes: 79 additions & 0 deletions
79
js_modules/dagster-ui/packages/ui-core/src/asset-graph/ComputeGraphData.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| import groupBy from 'lodash/groupBy'; | ||
|
|
||
| import {ComputeGraphDataMessageType} from './ComputeGraphData.types'; | ||
| import {GraphData, buildGraphData, toGraphId} from './Utils'; | ||
| import {AssetNodeForGraphQueryFragment} from './types/useAssetGraphData.types'; | ||
| import {GraphDataState} from './useAssetGraphData'; | ||
| import {filterAssetSelectionByQuery} from '../asset-selection/AntlrAssetSelection'; | ||
| import {doesFilterArrayMatchValueArray} from '../ui/Filters/doesFilterArrayMatchValueArray'; | ||
|
|
||
| export function computeGraphData({ | ||
| repoFilteredNodes, | ||
| graphQueryItems, | ||
| opsQuery, | ||
| kinds: _kinds, | ||
| hideEdgesToNodesOutsideQuery, | ||
| }: Omit<ComputeGraphDataMessageType, 'type'>): GraphDataState { | ||
| if (repoFilteredNodes === undefined || graphQueryItems === undefined) { | ||
| return { | ||
| allAssetKeys: [], | ||
| graphAssetKeys: [], | ||
| assetGraphData: null, | ||
| }; | ||
| } | ||
|
|
||
| // Filter the set of all AssetNodes down to those matching the `opsQuery`. | ||
| // In the future it might be ideal to move this server-side, but we currently | ||
| // get to leverage the useQuery cache almost 100% of the time above, making this | ||
| // super fast after the first load vs a network fetch on every page view. | ||
| const {all: allFilteredByOpQuery} = filterAssetSelectionByQuery(graphQueryItems, opsQuery); | ||
| const kinds = _kinds?.map((c) => c.toLowerCase()); | ||
| const all = kinds?.length | ||
| ? allFilteredByOpQuery.filter( | ||
| ({node}) => | ||
| node.kinds && | ||
| doesFilterArrayMatchValueArray( | ||
| kinds, | ||
| node.kinds.map((k) => k.toLowerCase()), | ||
| ), | ||
| ) | ||
| : allFilteredByOpQuery; | ||
|
|
||
| // Assemble the response into the data structure used for layout, traversal, etc. | ||
| const assetGraphData = buildGraphData(all.map((n) => n.node)); | ||
| if (hideEdgesToNodesOutsideQuery) { | ||
| removeEdgesToHiddenAssets(assetGraphData, repoFilteredNodes); | ||
| } | ||
|
|
||
| return { | ||
| allAssetKeys: repoFilteredNodes.map((n) => n.assetKey), | ||
| graphAssetKeys: all.map((n) => ({path: n.node.assetKey.path})), | ||
| assetGraphData, | ||
| }; | ||
| } | ||
|
|
||
| const removeEdgesToHiddenAssets = ( | ||
| graphData: GraphData, | ||
| allNodes: AssetNodeForGraphQueryFragment[], | ||
| ) => { | ||
| const allNodesById = groupBy(allNodes, (n) => toGraphId(n.assetKey)); | ||
| const notSourceAsset = (id: string) => !!allNodesById[id]; | ||
|
|
||
| for (const node of Object.keys(graphData.upstream)) { | ||
| for (const edge of Object.keys(graphData.upstream[node]!)) { | ||
| if (!graphData.nodes[edge] && notSourceAsset(node)) { | ||
| delete graphData.upstream[node]![edge]; | ||
| delete graphData.downstream[edge]![node]; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| for (const node of Object.keys(graphData.downstream)) { | ||
| for (const edge of Object.keys(graphData.downstream[node]!)) { | ||
| if (!graphData.nodes[edge] && notSourceAsset(node)) { | ||
| delete graphData.upstream[edge]![node]; | ||
| delete graphData.downstream[node]![edge]; | ||
| } | ||
| } | ||
| } | ||
| }; |
12 changes: 12 additions & 0 deletions
12
js_modules/dagster-ui/packages/ui-core/src/asset-graph/ComputeGraphData.types.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| import {AssetNodeForGraphQueryFragment} from './types/useAssetGraphData.types'; | ||
| import {AssetGraphFetchScope, AssetGraphQueryItem} from './useAssetGraphData'; | ||
|
|
||
| export type ComputeGraphDataMessageType = { | ||
| type: 'computeGraphData'; | ||
| repoFilteredNodes?: AssetNodeForGraphQueryFragment[]; | ||
| graphQueryItems?: AssetGraphQueryItem[]; | ||
| opsQuery: string; | ||
| kinds: AssetGraphFetchScope['kinds']; | ||
| hideEdgesToNodesOutsideQuery?: boolean; | ||
| flagAssetSelectionSyntax?: boolean; | ||
| }; |
25 changes: 25 additions & 0 deletions
25
js_modules/dagster-ui/packages/ui-core/src/asset-graph/ComputeGraphData.worker.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| import {FeatureFlag} from 'shared/app/FeatureFlags.oss'; | ||
|
|
||
| import {computeGraphData} from './ComputeGraphData'; | ||
| import {ComputeGraphDataMessageType} from './ComputeGraphData.types'; | ||
| import {setFeatureFlags} from '../app/Flags'; | ||
|
|
||
| type WorkerMessageData = ComputeGraphDataMessageType; | ||
|
|
||
| self.addEventListener('message', async (event: MessageEvent & {data: WorkerMessageData}) => { | ||
| const {data} = event; | ||
|
|
||
| if (data.type === 'computeGraphData') { | ||
| if (data.flagAssetSelectionSyntax) { | ||
| setFeatureFlags({[FeatureFlag.flagAssetSelectionSyntax]: true}); | ||
| } | ||
| const state = await computeGraphData(data); | ||
| self.postMessage(state); | ||
| } | ||
| }); | ||
|
|
||
| self.onmessage = function (event) { | ||
| if (event.data === 'close') { | ||
| self.close(); // Terminates the worker | ||
| } | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.