From d9ddad45f4a199787b54cb940cb630dbab2e458f Mon Sep 17 00:00:00 2001 From: Isaac Hellendag <2823852+hellendag@users.noreply.github.com> Date: Thu, 19 Dec 2024 15:11:01 -0600 Subject: [PATCH] [ui] Move stepStats out of RunRootQuery (#26610) ## Summary & Motivation Move `stepStats` out of the root Run query, since it can be expensive. Instead, query for it at the `RunMetadataProvider`. This should allow the header, Gantt chart, and logs to load even while the step stats panel of the page continues to load. ## How I Tested These Changes View a very long-running run. Verify that the Run header, Gantt chart, and logs appear fairly quickly, and the right panel takes a bit longer to load. ## Changelog [ui] Improve performance of Run page for very long-running runs. --- .../dagster-ui/packages/ui-core/client.json | 5 +- .../ui-core/src/runs/RunFragments.tsx | 14 ----- .../ui-core/src/runs/RunMetadataProvider.tsx | 60 +++++++++++++++++-- .../types/RunActionButtonsTestQuery.types.ts | 19 +----- .../src/runs/types/RunFragments.types.ts | 18 ------ .../runs/types/RunMetadataProvider.types.ts | 48 +++++++++++++++ .../ui-core/src/runs/types/RunRoot.types.ts | 19 +----- 7 files changed, 108 insertions(+), 75 deletions(-) diff --git a/js_modules/dagster-ui/packages/ui-core/client.json b/js_modules/dagster-ui/packages/ui-core/client.json index 9b8c430dd2ead..8e9e8ae0c4b40 100644 --- a/js_modules/dagster-ui/packages/ui-core/client.json +++ b/js_modules/dagster-ui/packages/ui-core/client.json @@ -123,7 +123,8 @@ "RunAssetChecksQuery": "6946372fc625c6aba249a54be1943c0858c8efbd5e6f5c64c55723494dc199e4", "RunAssetsQuery": "53c1e7814d451dfd58fb2427dcb326a1e9628c8bbc91b3b9c76f8d6c7b75e278", "RunTabsCountQuery": "5fe1760a3bf0494fb98e3c09f31add5138f9f31d59507a8b25186e2103bebbb4", - "RunRootQuery": "1aa4561b33c2cfb079d7a3ff284096fc3208a46dee748a24c7af827a2cb22919", + "RunStepStatsQuery": "77d73353a4aea095bfa241903122abf14eb38341c5869a9688b70c0d53f5a167", + "RunRootQuery": "4f2633b31ddc71c08d3a985be30dc1bf21fbc462287554f165060c51a3554beb", "RunStatsQuery": "75e80f740a79607de9e1152f9b7074d319197fbc219784c767c1abd5553e9a49", "LaunchPipelineExecution": "292088c4a697aca6be1d3bbc0cfc45d8a13cdb2e75cfedc64b68c6245ea34f89", "LaunchMultipleRuns": "a56d9efdb35e71e0fd1744dd768129248943bc5b23e717458b82c46829661763", @@ -135,7 +136,7 @@ "RunTagValuesQuery": "0c0a9998c215bb801eb0adcd5449c0ac4cf1e8efbc6d0fcc5fb6d76fcc95cb92", "ScheduledRunsListQuery": "2650d8ebdfc444fe76fcf8acd9ff54f9ecacdb680b1d83e3f487cb71dd0c7eae", "TerminateRunIdsQuery": "d38573af47f3ab2f2b11d90cb85ce8426307e2384e67a5b20e2bf67d5c1054bb", - "RunActionButtonsTestQuery": "d85a7e0201a27eb36be5a7471d2724fe5a68b7257e6635f54f120fc40f8169c0", + "RunActionButtonsTestQuery": "5d358c3360e683549b885108c3dbb7c1d21d8afd790a5ee963e6e9640ccdbfe8", "RunsRootQuery": "091646e47ecea81ba4765a3f2cead18880b09ee400d1d7e9dcb6e194ee364e51", "RunsFeedRootQuery": "ef8eb6ca144d661c6bcd409ed878551851f15dd1c0aa8c03ee9c68c1c4c301d1", "OngoingRunTimelineQuery": "055420e85ba799b294bab52c01d3f4a4470580606a40483031c35777d88d527f", diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/RunFragments.tsx b/js_modules/dagster-ui/packages/ui-core/src/runs/RunFragments.tsx index c0ebb65c183fd..b34d76e07dab3 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/RunFragments.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/RunFragments.tsx @@ -46,20 +46,6 @@ export const RUN_FRAGMENT = gql` } stepKeysToExecute updateTime - stepStats { - stepKey - status - startTime - endTime - attempts { - startTime - endTime - } - markers { - startTime - endTime - } - } ...RunTimingFragment } diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/RunMetadataProvider.tsx b/js_modules/dagster-ui/packages/ui-core/src/runs/RunMetadataProvider.tsx index c99bfc4106a6f..a3d495d021e7e 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/RunMetadataProvider.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/RunMetadataProvider.tsx @@ -1,11 +1,17 @@ import * as React from 'react'; +import {useMemo} from 'react'; import {LogsProviderLogs} from './LogsProvider'; import {RunContext} from './RunContext'; -import {gql} from '../apollo-client'; +import {gql, useQuery} from '../apollo-client'; import {flattenOneLevel} from '../util/flattenOneLevel'; import {RunFragment} from './types/RunFragments.types'; -import {RunMetadataProviderMessageFragment} from './types/RunMetadataProvider.types'; +import { + RunMetadataProviderMessageFragment, + RunStepStatsFragment, + RunStepStatsQuery, + RunStepStatsQueryVariables, +} from './types/RunMetadataProvider.types'; import {StepEventStatus} from '../graphql/types'; import {METADATA_ENTRY_FRAGMENT} from '../metadata/MetadataEntryFragment'; @@ -103,13 +109,17 @@ export const extractLogCaptureStepsFromLegacySteps = (stepKeys: string[]) => { const fromTimestamp = (ts: number | null) => (ts ? Math.floor(ts * 1000) : undefined); -function extractMetadataFromRun(run?: RunFragment): IRunMetadataDict { +function extractMetadataFromRun( + run: RunFragment | null = null, + stepStats: RunStepStatsFragment['stepStats'] = [], +): IRunMetadataDict { const metadata: IRunMetadataDict = { firstLogAt: 0, mostRecentLogAt: 0, globalMarkers: [], steps: {}, }; + if (!run) { return metadata; } @@ -120,7 +130,7 @@ function extractMetadataFromRun(run?: RunFragment): IRunMetadataDict { metadata.exitedAt = fromTimestamp(run.endTime); } - run.stepStats.forEach((stepStat) => { + stepStats.forEach((stepStat) => { metadata.steps[stepStat.stepKey] = { // state: // current state @@ -370,7 +380,18 @@ interface IRunMetadataProviderProps { export const RunMetadataProvider = ({logs, children}: IRunMetadataProviderProps) => { const run = React.useContext(RunContext); - const runMetadata = React.useMemo(() => extractMetadataFromRun(run), [run]); + + // Step stats can be expensive to load, so we separate them from the main run query. + const {data} = useQuery(RUN_STEP_STATS_QUERY, { + variables: run ? {runId: run.id} : undefined, + skip: !run, + }); + + const stepStats = useMemo(() => { + return data?.pipelineRunOrError.__typename === 'Run' ? data.pipelineRunOrError.stepStats : []; + }, [data]); + + const runMetadata = React.useMemo(() => extractMetadataFromRun(run, stepStats), [run, stepStats]); const metadata = React.useMemo( () => logs.loading ? runMetadata : extractMetadataFromLogs(flattenOneLevel(logs.allNodeChunks)), @@ -379,6 +400,35 @@ export const RunMetadataProvider = ({logs, children}: IRunMetadataProviderProps) return <>{children(metadata)}; }; +const RUN_STEP_STATS_QUERY = gql` + query RunStepStatsQuery($runId: ID!) { + pipelineRunOrError(runId: $runId) { + ... on Run { + id + ...RunStepStatsFragment + } + } + } + + fragment RunStepStatsFragment on Run { + id + stepStats { + stepKey + status + startTime + endTime + attempts { + startTime + endTime + } + markers { + startTime + endTime + } + } + } +`; + export const RUN_METADATA_PROVIDER_MESSAGE_FRAGMENT = gql` fragment RunMetadataProviderMessageFragment on DagsterRunEvent { ... on MessageEvent { diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/__tests__/types/RunActionButtonsTestQuery.types.ts b/js_modules/dagster-ui/packages/ui-core/src/runs/__tests__/types/RunActionButtonsTestQuery.types.ts index ea344cf423ea4..cc5a89e67c1fb 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/__tests__/types/RunActionButtonsTestQuery.types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/__tests__/types/RunActionButtonsTestQuery.types.ts @@ -56,25 +56,8 @@ export type RunActionButtonsTestQuery = { }>; }>; } | null; - stepStats: Array<{ - __typename: 'RunStepStats'; - stepKey: string; - status: Types.StepEventStatus | null; - startTime: number | null; - endTime: number | null; - attempts: Array<{ - __typename: 'RunMarker'; - startTime: number | null; - endTime: number | null; - }>; - markers: Array<{ - __typename: 'RunMarker'; - startTime: number | null; - endTime: number | null; - }>; - }>; } | {__typename: 'RunNotFoundError'}; }; -export const RunActionButtonsTestQueryVersion = 'd85a7e0201a27eb36be5a7471d2724fe5a68b7257e6635f54f120fc40f8169c0'; +export const RunActionButtonsTestQueryVersion = '5d358c3360e683549b885108c3dbb7c1d21d8afd790a5ee963e6e9640ccdbfe8'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunFragments.types.ts b/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunFragments.types.ts index a50b77c1b0de8..e75aab9ccde18 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunFragments.types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunFragments.types.ts @@ -49,15 +49,6 @@ export type RunFragment = { }>; }>; } | null; - stepStats: Array<{ - __typename: 'RunStepStats'; - stepKey: string; - status: Types.StepEventStatus | null; - startTime: number | null; - endTime: number | null; - attempts: Array<{__typename: 'RunMarker'; startTime: number | null; endTime: number | null}>; - markers: Array<{__typename: 'RunMarker'; startTime: number | null; endTime: number | null}>; - }>; }; export type RunDagsterRunEventFragment_AlertFailureEvent = { @@ -3186,13 +3177,4 @@ export type RunPageFragment = { }>; }>; } | null; - stepStats: Array<{ - __typename: 'RunStepStats'; - stepKey: string; - status: Types.StepEventStatus | null; - startTime: number | null; - endTime: number | null; - attempts: Array<{__typename: 'RunMarker'; startTime: number | null; endTime: number | null}>; - markers: Array<{__typename: 'RunMarker'; startTime: number | null; endTime: number | null}>; - }>; }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunMetadataProvider.types.ts b/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunMetadataProvider.types.ts index db950cff62923..9c7d0454624ba 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunMetadataProvider.types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunMetadataProvider.types.ts @@ -2,6 +2,52 @@ import * as Types from '../../graphql/types'; +export type RunStepStatsQueryVariables = Types.Exact<{ + runId: Types.Scalars['ID']['input']; +}>; + +export type RunStepStatsQuery = { + __typename: 'Query'; + pipelineRunOrError: + | {__typename: 'PythonError'} + | { + __typename: 'Run'; + id: string; + stepStats: Array<{ + __typename: 'RunStepStats'; + stepKey: string; + status: Types.StepEventStatus | null; + startTime: number | null; + endTime: number | null; + attempts: Array<{ + __typename: 'RunMarker'; + startTime: number | null; + endTime: number | null; + }>; + markers: Array<{ + __typename: 'RunMarker'; + startTime: number | null; + endTime: number | null; + }>; + }>; + } + | {__typename: 'RunNotFoundError'}; +}; + +export type RunStepStatsFragment = { + __typename: 'Run'; + id: string; + stepStats: Array<{ + __typename: 'RunStepStats'; + stepKey: string; + status: Types.StepEventStatus | null; + startTime: number | null; + endTime: number | null; + attempts: Array<{__typename: 'RunMarker'; startTime: number | null; endTime: number | null}>; + markers: Array<{__typename: 'RunMarker'; startTime: number | null; endTime: number | null}>; + }>; +}; + export type RunMetadataProviderMessageFragment_AlertFailureEvent = { __typename: 'AlertFailureEvent'; message: string; @@ -488,3 +534,5 @@ export type RunMetadataProviderMessageFragment = | RunMetadataProviderMessageFragment_StepExpectationResultEvent | RunMetadataProviderMessageFragment_StepWorkerStartedEvent | RunMetadataProviderMessageFragment_StepWorkerStartingEvent; + +export const RunStepStatsQueryVersion = '77d73353a4aea095bfa241903122abf14eb38341c5869a9688b70c0d53f5a167'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunRoot.types.ts b/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunRoot.types.ts index d32da61f71afd..8c196998cc90a 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunRoot.types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunRoot.types.ts @@ -58,25 +58,8 @@ export type RunRootQuery = { }>; }>; } | null; - stepStats: Array<{ - __typename: 'RunStepStats'; - stepKey: string; - status: Types.StepEventStatus | null; - startTime: number | null; - endTime: number | null; - attempts: Array<{ - __typename: 'RunMarker'; - startTime: number | null; - endTime: number | null; - }>; - markers: Array<{ - __typename: 'RunMarker'; - startTime: number | null; - endTime: number | null; - }>; - }>; } | {__typename: 'RunNotFoundError'}; }; -export const RunRootQueryVersion = '1aa4561b33c2cfb079d7a3ff284096fc3208a46dee748a24c7af827a2cb22919'; +export const RunRootQueryVersion = '4f2633b31ddc71c08d3a985be30dc1bf21fbc462287554f165060c51a3554beb';