diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/LogsProvider.tsx b/js_modules/dagster-ui/packages/ui-core/src/runs/LogsProvider.tsx index 190bc51f035a0..96ff886b987af 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/LogsProvider.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/LogsProvider.tsx @@ -39,7 +39,7 @@ export interface LogFilter { } export interface LogsProviderLogs { - allNodes: LogNode[]; + allNodeChunks: LogNode[][]; counts: LogLevelCounts; loading: boolean; } @@ -72,7 +72,7 @@ const BATCH_INTERVAL = 100; const QUERY_LOG_LIMIT = 1000; type State = { - nodes: LogNode[]; + nodeChunks: LogNode[][]; cursor: string | null; counts: LogLevelCounts; loading: boolean; @@ -99,25 +99,29 @@ const reducer = (state: State, action: Action) => { ...node, clientsideKey: `csk${node.timestamp}-${ii}`, })); - const nodes = [...state.nodes, ...queuedNodes]; + + const copy = state.nodeChunks.slice(); + copy.push(queuedNodes); + const counts = {...state.counts}; queuedNodes.forEach((node) => { const level = logNodeLevel(node); counts[level]++; }); - return {nodes, counts, loading: action.hasMore, cursor: action.cursor}; + + return {nodeChunks: copy, counts, loading: action.hasMore, cursor: action.cursor}; } case 'set-cursor': return {...state, cursor: action.cursor}; case 'reset': - return {nodes: [], counts: emptyCounts, cursor: null, loading: true}; + return {nodeChunks: [], counts: emptyCounts, cursor: null, loading: true}; default: return state; } }; const initialState: State = { - nodes: [], + nodeChunks: [] as LogNode[][], counts: emptyCounts, cursor: null, loading: true, @@ -184,7 +188,7 @@ const useLogsProviderWithSubscription = (runId: string) => { }, BATCH_INTERVAL); }, [syncPipelineStatusToApolloCache]); - const {nodes, counts, cursor, loading} = state; + const {nodeChunks, counts, cursor, loading} = state; const {availability, disabled, status} = React.useContext(WebSocketContext); const lostWebsocket = !disabled && availability === 'available' && status === WebSocket.CLOSED; @@ -227,10 +231,10 @@ const useLogsProviderWithSubscription = (runId: string) => { return React.useMemo( () => - nodes !== null - ? {allNodes: nodes, counts, loading, subscriptionComponent} - : {allNodes: [], counts, loading, subscriptionComponent}, - [counts, loading, nodes, subscriptionComponent], + nodeChunks !== null + ? {allNodeChunks: nodeChunks, counts, loading, subscriptionComponent} + : {allNodeChunks: [], counts, loading, subscriptionComponent}, + [counts, loading, nodeChunks, subscriptionComponent], ); }; @@ -285,7 +289,7 @@ const POLL_INTERVAL = 5000; const LogsProviderWithQuery = (props: LogsProviderWithQueryProps) => { const {children, runId} = props; const [state, dispatch] = React.useReducer(reducer, initialState); - const {counts, cursor, nodes} = state; + const {counts, cursor, nodeChunks} = state; const dependency = useTraceDependency('RunLogsQuery'); @@ -332,9 +336,9 @@ const LogsProviderWithQuery = (props: LogsProviderWithQueryProps) => { return ( <> {children( - nodes !== null && nodes.length > 0 - ? {allNodes: nodes, counts, loading: false} - : {allNodes: [], counts, loading: true}, + nodeChunks !== null && nodeChunks.length > 0 + ? {allNodeChunks: nodeChunks, counts, loading: false} + : {allNodeChunks: [], counts, loading: true}, )} ); @@ -350,7 +354,7 @@ export const LogsProvider = (props: LogsProviderProps) => { } if (availability === 'attempting-to-connect') { - return <>{children({allNodes: [], counts: emptyCounts, loading: true})}; + return <>{children({allNodeChunks: [], counts: emptyCounts, loading: true})}; } return {children}; diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/LogsScrollingTable.tsx b/js_modules/dagster-ui/packages/ui-core/src/runs/LogsScrollingTable.tsx index 35ff1f61b5c7a..a89eac0f5b8c6 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/LogsScrollingTable.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/LogsScrollingTable.tsx @@ -1,4 +1,4 @@ -import {Box, Colors, NonIdealState, Row} from '@dagster-io/ui-components'; +import {Box, Colors, NonIdealState, Row, SpinnerWithText} from '@dagster-io/ui-components'; import {useVirtualizer} from '@tanstack/react-virtual'; import {useEffect, useRef} from 'react'; import styled from 'styled-components'; @@ -81,12 +81,10 @@ export const LogsScrollingTable = (props: Props) => { }, [totalHeight, virtualizer]); const content = () => { - if (logs.loading) { + if (logs.allNodeChunks.length === 0 && logs.loading) { return ( - - - - + + ); } diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/RunMetadataProvider.tsx b/js_modules/dagster-ui/packages/ui-core/src/runs/RunMetadataProvider.tsx index b6f2f269f0068..c99bfc4106a6f 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/RunMetadataProvider.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/RunMetadataProvider.tsx @@ -3,6 +3,7 @@ import * as React from 'react'; import {LogsProviderLogs} from './LogsProvider'; import {RunContext} from './RunContext'; import {gql} from '../apollo-client'; +import {flattenOneLevel} from '../util/flattenOneLevel'; import {RunFragment} from './types/RunFragments.types'; import {RunMetadataProviderMessageFragment} from './types/RunMetadataProvider.types'; import {StepEventStatus} from '../graphql/types'; @@ -371,7 +372,8 @@ export const RunMetadataProvider = ({logs, children}: IRunMetadataProviderProps) const run = React.useContext(RunContext); const runMetadata = React.useMemo(() => extractMetadataFromRun(run), [run]); const metadata = React.useMemo( - () => (logs.loading ? runMetadata : extractMetadataFromLogs(logs.allNodes)), + () => + logs.loading ? runMetadata : extractMetadataFromLogs(flattenOneLevel(logs.allNodeChunks)), [logs, runMetadata], ); return <>{children(metadata)}; diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/StepLogsDialog.tsx b/js_modules/dagster-ui/packages/ui-core/src/runs/StepLogsDialog.tsx index b8506ab6a6f2c..4f107eb164302 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/StepLogsDialog.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/StepLogsDialog.tsx @@ -8,7 +8,7 @@ import { Mono, Spinner, } from '@dagster-io/ui-components'; -import {useState} from 'react'; +import {useMemo, useState} from 'react'; import * as React from 'react'; import {Link} from 'react-router-dom'; import styled from 'styled-components'; @@ -22,6 +22,7 @@ import {IRunMetadataDict, RunMetadataProvider} from './RunMetadataProvider'; import {titleForRun} from './RunUtils'; import {useComputeLogFileKeyForSelection} from './useComputeLogFileKeyForSelection'; import {DagsterEventType} from '../graphql/types'; +import {flattenOneLevel} from '../util/flattenOneLevel'; export function useStepLogs({runId, stepKeys}: {runId?: string; stepKeys?: string[]}) { const [showingLogs, setShowingLogs] = React.useState<{runId: string; stepKeys: string[]} | null>( @@ -113,9 +114,12 @@ export const StepLogsModalContent = ({ const [logType, setComputeLogType] = useState(LogType.structured); const [computeLogUrl, setComputeLogUrl] = React.useState(null); - const firstLogForStep = logs.allNodes.find( + const flatLogs = useMemo(() => flattenOneLevel(logs.allNodeChunks), [logs]); + + const firstLogForStep = flatLogs.find( (l) => l.eventType === DagsterEventType.STEP_START && l.stepKey && stepKeys.includes(l.stepKey), ); + const firstLogForStepTime = firstLogForStep ? Number(firstLogForStep.timestamp) : 0; const [filter, setFilter] = useState({ diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/filterLogs.tsx b/js_modules/dagster-ui/packages/ui-core/src/runs/filterLogs.tsx index 368fadca1b641..0882ed53b3181 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/filterLogs.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/filterLogs.tsx @@ -3,9 +3,10 @@ import {eventTypeToDisplayType} from './getRunFilterProviders'; import {logNodeLevel} from './logNodeLevel'; import {LogNode} from './types'; import {weakmapMemoize} from '../app/Util'; +import {flattenOneLevel} from '../util/flattenOneLevel'; export function filterLogs(logs: LogsProviderLogs, filter: LogFilter, filterStepKeys: string[]) { - const filteredNodes = logs.allNodes.filter((node) => { + const filteredNodes = flattenOneLevel(logs.allNodeChunks).filter((node) => { // These events are used to determine which assets a run will materialize and are not intended // to be displayed in the Dagster UI. Pagination is offset based, so we remove these logs client-side. if ( diff --git a/js_modules/dagster-ui/packages/ui-core/src/util/flattenOneLevel.tsx b/js_modules/dagster-ui/packages/ui-core/src/util/flattenOneLevel.tsx new file mode 100644 index 0000000000000..18c8b12347bd5 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/util/flattenOneLevel.tsx @@ -0,0 +1,10 @@ +/** + * Flattens a two-dimensional array into a one-dimensional array. + * + * @param nodeChunks - The two-dimensional array to flatten. + * @returns The flattened one-dimensional array. + */ +// https://jsbench.me/o8kqzo8olz/1 +export function flattenOneLevel(arrays: T[][]) { + return ([] as T[]).concat(...arrays); +}