From 1d57224ee0bcfeead5d4143bc9a032aca0d44e7f Mon Sep 17 00:00:00 2001 From: David Liu <48995019+dliu27@users.noreply.github.com> Date: Mon, 9 Dec 2024 12:41:23 -0500 Subject: [PATCH] [6/n] [RFC] add launch all frontend functionality for schedules (#26063) ## Summary & Motivation Linear: https://linear.app/dagster-labs/issue/FE-654/[fe]-implement-launch-all-frontend-schedule Adds the frontend for 'launch all' for manual tick schedules Video: https://github.com/user-attachments/assets/1d3a9315-cc05-4142-8964-5e86e077403b ## How I Tested These Changes tested the launch all runs flow locally yarn lint, ts, jest, generate-graphql in ui-core --- .../useLaunchMultipleRunsWithTelemetry.ts | 31 +- .../src/ticks/EvaluateScheduleDialog.tsx | 422 ++++++++++++------ .../src/ticks/EvaluateTickButtonSchedule.tsx | 1 - .../__tests__/EvaluateScheduleDialog.test.tsx | 3 + .../src/util/buildExecutionParamsList.ts | 45 +- 5 files changed, 359 insertions(+), 143 deletions(-) diff --git a/js_modules/dagster-ui/packages/ui-core/src/launchpad/useLaunchMultipleRunsWithTelemetry.ts b/js_modules/dagster-ui/packages/ui-core/src/launchpad/useLaunchMultipleRunsWithTelemetry.ts index f79ab12cea078..9317a6c74597a 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/launchpad/useLaunchMultipleRunsWithTelemetry.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/launchpad/useLaunchMultipleRunsWithTelemetry.ts @@ -25,23 +25,26 @@ export function useLaunchMultipleRunsWithTelemetry() { return useCallback( async (variables: LaunchMultipleRunsMutationVariables, behavior: LaunchBehavior) => { - const executionParamsList = Array.isArray(variables.executionParamsList) - ? variables.executionParamsList - : [variables.executionParamsList]; - const jobNames = executionParamsList.map((params) => params.selector?.jobName); + try { + const executionParamsList = Array.isArray(variables.executionParamsList) + ? variables.executionParamsList + : [variables.executionParamsList]; + const jobNames = executionParamsList.map((params) => params.selector?.jobName); - if (jobNames.length !== executionParamsList.length || jobNames.includes(undefined)) { - return; - } + if ( + jobNames.length !== executionParamsList.length || + jobNames.includes(undefined) || + jobNames.includes(null) + ) { + throw new Error('Invalid job names'); + } - const metadata: {[key: string]: string | string[] | null | undefined} = { - jobNames: jobNames.filter((name): name is string => name !== undefined), - opSelection: undefined, - }; + const metadata: {[key: string]: string | string[] | null | undefined} = { + jobNames: jobNames.filter((name): name is string => name !== undefined), + opSelection: undefined, + }; - let result; - try { - result = (await launchMultipleRuns({variables})).data?.launchMultipleRuns; + const result = (await launchMultipleRuns({variables})).data?.launchMultipleRuns; if (result) { handleLaunchMultipleResult(result, history, {behavior}); logTelemetry( diff --git a/js_modules/dagster-ui/packages/ui-core/src/ticks/EvaluateScheduleDialog.tsx b/js_modules/dagster-ui/packages/ui-core/src/ticks/EvaluateScheduleDialog.tsx index b8249d6a81275..ddfd6b42faba7 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/ticks/EvaluateScheduleDialog.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/ticks/EvaluateScheduleDialog.tsx @@ -14,9 +14,10 @@ import { Spinner, Subheading, Tag, + Tooltip, useViewport, } from '@dagster-io/ui-components'; -import {useContext, useEffect, useMemo, useRef, useState} from 'react'; +import {useCallback, useContext, useMemo, useRef, useState} from 'react'; import styled from 'styled-components'; import {RunRequestTable} from './DryRunRequestTable'; @@ -28,14 +29,25 @@ import { ScheduleDryRunMutation, ScheduleDryRunMutationVariables, } from './types/EvaluateScheduleDialog.types'; +import {showCustomAlert} from '../app/CustomAlertProvider'; import {PYTHON_ERROR_FRAGMENT} from '../app/PythonErrorFragment'; import {PythonErrorInfo} from '../app/PythonErrorInfo'; +import {assertUnreachable} from '../app/Util'; import {TimeContext} from '../app/time/TimeContext'; import {timestampToString} from '../app/time/timestampToString'; +import {PythonErrorFragment} from '../app/types/PythonErrorFragment.types'; +import {ScheduleSelector} from '../graphql/types'; +import {useLaunchMultipleRunsWithTelemetry} from '../launchpad/useLaunchMultipleRunsWithTelemetry'; import {testId} from '../testing/testId'; +import {buildExecutionParamsListSchedule} from '../util/buildExecutionParamsList'; import {repoAddressToSelector} from '../workspace/repoAddressToSelector'; import {RepoAddress} from '../workspace/types'; +export type ScheduleDryRunInstigationTick = Extract< + ScheduleDryRunMutation['scheduleDryRun'], + {__typename: 'DryRunInstigationTick'} +>; + const locale = navigator.language; type Props = { @@ -64,132 +76,315 @@ export const EvaluateScheduleDialog = (props: Props) => { }; const EvaluateSchedule = ({repoAddress, name, onClose, jobName}: Props) => { - const [_selectedTimestamp, setSelectedTimestamp] = useState<{ts: number; label: string}>(); - const {data} = useQuery(GET_SCHEDULE_QUERY, { - variables: { - scheduleSelector: { - repositoryLocationName: repoAddress.location, - repositoryName: repoAddress.name, - scheduleName: name, + const [selectedTimestamp, setSelectedTimestamp] = useState<{ts: number; label: string}>(); + const scheduleSelector: ScheduleSelector = useMemo( + () => ({ + repositoryLocationName: repoAddress.location, + repositoryName: repoAddress.name, + scheduleName: name, + }), + [repoAddress, name], + ); + + // query to get the schedule initially + const {data: getScheduleData} = useQuery( + GET_SCHEDULE_QUERY, + { + variables: { + scheduleSelector, }, }, - }); + ); + + // mutation to evaluate the schedule + const [scheduleDryRunMutation, {loading: scheduleDryRunMutationLoading}] = useMutation< + ScheduleDryRunMutation, + ScheduleDryRunMutationVariables + >(SCHEDULE_DRY_RUN_MUTATION); + + // mutation to launch all runs + const launchMultipleRunsWithTelemetry = useLaunchMultipleRunsWithTelemetry(); + const { timezone: [userTimezone], } = useContext(TimeContext); const [isTickSelectionOpen, setIsTickSelectionOpen] = useState(false); const selectedTimestampRef = useRef<{ts: number; label: string} | null>(null); const {viewport, containerProps} = useViewport(); - const [shouldEvaluate, setShouldEvaluate] = useState(false); + const [launching, setLaunching] = useState(false); + + const [scheduleExecutionError, setScheduleExecutionError] = useState( + null, + ); + const [scheduleExecutionData, setScheduleExecutionData] = + useState(null); + + const canSubmitTest = useMemo(() => { + return getScheduleData && !scheduleDryRunMutationLoading; + }, [getScheduleData, scheduleDryRunMutationLoading]); + + // handle clicking Evaluate button + const submitTest = useCallback(async () => { + if (!canSubmitTest) { + return; + } + + const repositorySelector = repoAddressToSelector(repoAddress); + + const result = await scheduleDryRunMutation({ + variables: { + selectorData: { + ...repositorySelector, + scheduleName: name, + }, + timestamp: selectedTimestampRef.current!.ts, + }, + }); + + const data = result.data?.scheduleDryRun; + + if (data) { + if (data?.__typename === 'DryRunInstigationTick') { + if (data.evaluationResult?.error) { + setScheduleExecutionError(data.evaluationResult.error); + } else { + setScheduleExecutionData(data); + } + } else if (data?.__typename === 'ScheduleNotFoundError') { + showCustomAlert({ + title: 'Schedule not found', + body: `Could not find a schedule named: ${name}`, + }); + } else { + setScheduleExecutionError(data); + } + } else { + assertUnreachable('scheduleDryRun Mutation returned no data??' as never); + } + }, [canSubmitTest, scheduleDryRunMutation, repoAddress, name]); + + const executionParamsList = useMemo( + () => + scheduleExecutionData && scheduleSelector + ? buildExecutionParamsListSchedule(scheduleExecutionData, scheduleSelector) + : [], + [scheduleSelector, scheduleExecutionData], + ); + + const canLaunchAll = useMemo(() => { + return executionParamsList != null && executionParamsList.length > 0; + }, [executionParamsList]); + + // handle clicking Launch all button + const onLaunchAll = useCallback(async () => { + if (!canLaunchAll) { + return; + } + setLaunching(true); + + try { + if (executionParamsList) { + await launchMultipleRunsWithTelemetry({executionParamsList}, 'toast'); + } + } catch (e) { + console.error(e); + } + + setLaunching(false); + onClose(); + }, [canLaunchAll, executionParamsList, launchMultipleRunsWithTelemetry, onClose]); + const content = useMemo(() => { - if (shouldEvaluate) { + // launching all runs state + if (launching) { + return ( + + +
Launching runs
+ + ); + } + + // initial loading state when schedule data hasn't been queried yet + if (!getScheduleData) { return ( - + + + ); + } + + // error states after getting schedule data + if (getScheduleData.scheduleOrError.__typename === 'PythonError') { + return ; + } + + if (getScheduleData.scheduleOrError.__typename === 'ScheduleNotFoundError') { + return ( + + ); + } + + // handle showing results page after clicking Evaluate + if (scheduleExecutionData || scheduleExecutionError) { + return ( + ); } - if (!data) { + + // loading state for evaluating + if (scheduleDryRunMutationLoading) { return ( - - + + +
Evaluating schedule
 ); + } else { + // tick selection page + const timestamps = getScheduleData.scheduleOrError.potentialTickTimestamps.map((ts) => ({ + ts, + label: timestampToString({ + timestamp: {unix: ts}, + locale, + timezone: userTimezone, + timeFormat: { + showTimezone: true, + }, + }), + })); + selectedTimestampRef.current = selectedTimestamp || timestamps[0] || null; + return ( +
+ Select a mock evaluation time + + {timestamps.map((timestamp) => ( + {timestamp.label}
} + onClick={() => { + setSelectedTimestamp(timestamp); + setIsTickSelectionOpen(false); + }} + /> + ))} + + } + > +
+ +
+ + + ); } - if (data.scheduleOrError.__typename === 'PythonError') { - return
; - } - if (data.scheduleOrError.__typename === 'ScheduleNotFoundError') { - return
; - } - const timestamps = data.scheduleOrError.potentialTickTimestamps.map((ts) => ({ - ts, - label: timestampToString({ - timestamp: {unix: ts}, - locale, - timezone: userTimezone, - timeFormat: { - showTimezone: true, - }, - }), - })); - selectedTimestampRef.current = _selectedTimestamp || timestamps[0] || null; - return ( -
- Select a mock evaluation time - - {timestamps.map((timestamp) => ( - {timestamp.label}
} - onClick={() => { - setSelectedTimestamp(timestamp); - setIsTickSelectionOpen(false); - }} - /> - ))} - - } - > -
- -
- -
- ); }, [ - _selectedTimestamp, - containerProps, - data, - isTickSelectionOpen, - jobName, - name, + launching, + getScheduleData, + scheduleExecutionData, + scheduleExecutionError, + scheduleDryRunMutationLoading, repoAddress, - shouldEvaluate, - userTimezone, + name, + jobName, + selectedTimestamp, + isTickSelectionOpen, viewport.width, + containerProps, + userTimezone, ]); const buttons = useMemo(() => { - if (!shouldEvaluate) { + if (launching) { + return ; + } + + if (scheduleExecutionData || scheduleExecutionError) { + return ( + + + + + + + + + ); + } + + if (scheduleDryRunMutationLoading) { + return ( + + + + ); + } else { return ( <> ); - } else { - return ( - <> - - - - ); } - }, [onClose, shouldEvaluate]); + }, [ + canLaunchAll, + canSubmitTest, + launching, + onClose, + onLaunchAll, + scheduleExecutionData, + scheduleExecutionError, + submitTest, + scheduleDryRunMutationLoading, + ]); return ( <> @@ -226,62 +421,34 @@ export const GET_SCHEDULE_QUERY = gql` } `; -const EvaluateScheduleContent = ({ +// FE for showing result of evaluating schedule (error, skipped, or success state) +const EvaluateScheduleResult = ({ repoAddress, name, timestamp, jobName, + scheduleExecutionData, + scheduleExecutionError, }: { repoAddress: RepoAddress; name: string; timestamp: number; jobName: string; + scheduleExecutionData: ScheduleDryRunInstigationTick | null; + scheduleExecutionError: PythonErrorFragment | null; }) => { const { timezone: [userTimezone], } = useContext(TimeContext); - const [scheduleDryRunMutation] = useMutation< - ScheduleDryRunMutation, - ScheduleDryRunMutationVariables - >( - SCHEDULE_DRY_RUN_MUTATION, - useMemo(() => { - const repositorySelector = repoAddressToSelector(repoAddress); - return { - variables: { - selectorData: { - ...repositorySelector, - scheduleName: name, - }, - timestamp, - }, - }; - }, [name, repoAddress, timestamp]), - ); - const [result, setResult] = useState> | null>( - null, - ); - useEffect(() => { - scheduleDryRunMutation().then((result) => { - setResult(() => result); - }); - }, [scheduleDryRunMutation]); - - if (!result || !result.data) { - return ( - - - - ); - } - const evaluationResult = - result?.data?.scheduleDryRun.__typename === 'DryRunInstigationTick' - ? result?.data?.scheduleDryRun.evaluationResult - : null; + const evaluationResult = scheduleExecutionData?.evaluationResult; const innerContent = () => { - const data = result.data; + if (scheduleExecutionError) { + return ; + } + + const data = scheduleExecutionData; if (!data || !evaluationResult) { return ( @@ -363,6 +530,7 @@ const EvaluateScheduleContent = ({ ); }; + export const SCHEDULE_DRY_RUN_MUTATION = gql` mutation ScheduleDryRunMutation($selectorData: ScheduleSelector!, $timestamp: Float) { scheduleDryRun(selectorData: $selectorData, timestamp: $timestamp) { diff --git a/js_modules/dagster-ui/packages/ui-core/src/ticks/EvaluateTickButtonSchedule.tsx b/js_modules/dagster-ui/packages/ui-core/src/ticks/EvaluateTickButtonSchedule.tsx index 03ea49e3fdf2d..1929282a95139 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/ticks/EvaluateTickButtonSchedule.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/ticks/EvaluateTickButtonSchedule.tsx @@ -27,7 +27,6 @@ export const EvaluateTickButtonSchedule = ({ Evaluate tick { setShowTestTickDialog(false); diff --git a/js_modules/dagster-ui/packages/ui-core/src/ticks/__tests__/EvaluateScheduleDialog.test.tsx b/js_modules/dagster-ui/packages/ui-core/src/ticks/__tests__/EvaluateScheduleDialog.test.tsx index c7970d1fba44c..c9406fcfc43f4 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/ticks/__tests__/EvaluateScheduleDialog.test.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/ticks/__tests__/EvaluateScheduleDialog.test.tsx @@ -61,6 +61,9 @@ describe('EvaluateScheduleTest', () => { expect(screen.getByTestId('tick-5')).toBeVisible(); }); await userEvent.click(screen.getByTestId('tick-5')); + await waitFor(() => { + expect(screen.getByTestId('evaluate')).not.toBeDisabled(); + }); await userEvent.click(screen.getByTestId('evaluate')); await waitFor(() => { expect(screen.getByText('Failed')).toBeVisible(); diff --git a/js_modules/dagster-ui/packages/ui-core/src/util/buildExecutionParamsList.ts b/js_modules/dagster-ui/packages/ui-core/src/util/buildExecutionParamsList.ts index b58e58c7b7611..d5264bf96a6b6 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/util/buildExecutionParamsList.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/util/buildExecutionParamsList.ts @@ -1,8 +1,9 @@ import * as yaml from 'yaml'; import {showCustomAlert} from '../app/CustomAlertProvider'; -import {ExecutionParams, SensorSelector} from '../graphql/types'; +import {ExecutionParams, ScheduleSelector, SensorSelector} from '../graphql/types'; import {sanitizeConfigYamlString} from '../launchpad/yamlUtils'; +import {ScheduleDryRunInstigationTick} from '../ticks/EvaluateScheduleDialog'; import {SensorDryRunInstigationTick} from '../ticks/SensorDryRunDialog'; const YAML_SYNTAX_INVALID = `The YAML you provided couldn't be parsed. Please fix the syntax errors and try again.`; @@ -51,3 +52,45 @@ export const buildExecutionParamsListSensor = ( }); return executionParamsList; }; + +// adapted from buildExecutionVariables() in LaunchpadSession.tsx +export const buildExecutionParamsListSchedule = ( + scheduleExecutionData: ScheduleDryRunInstigationTick, + scheduleSelector: ScheduleSelector, +) => { + if (!scheduleExecutionData) { + return []; + } + + const executionParamsList: ExecutionParams[] = []; + + scheduleExecutionData?.evaluationResult?.runRequests?.forEach((request) => { + const configYamlOrEmpty = sanitizeConfigYamlString(request.runConfigYaml); + + try { + yaml.parse(configYamlOrEmpty); + } catch { + showCustomAlert({title: 'Invalid YAML', body: YAML_SYNTAX_INVALID}); + return; + } + const {repositoryLocationName, repositoryName} = scheduleSelector; + + const executionParams: ExecutionParams = { + runConfigData: configYamlOrEmpty, + selector: { + jobName: request.jobName, // get jobName from runRequest + repositoryLocationName, + repositoryName, + assetSelection: [], + assetCheckSelection: [], + solidSelection: undefined, + }, + mode: 'default', + executionMetadata: { + tags: [...request.tags.map(onlyKeyAndValue)], + }, + }; + executionParamsList.push(executionParams); + }); + return executionParamsList; +};