[ui] Fix endless log loading

js_modules/dagster-ui/packages/ui-core/src/runs/LogsProvider.tsx

@@ -39,7 +39,7 @@ export interface LogFilter {
 }
 
 export interface LogsProviderLogs {
-  allNodes: LogNode[];
+  allNodeChunks: LogNode[][];
   counts: LogLevelCounts;
   loading: boolean;
 }
@@ -72,7 +72,7 @@ const BATCH_INTERVAL = 100;
 const QUERY_LOG_LIMIT = 1000;
 
 type State = {
-  nodes: LogNode[];
+  nodeChunks: LogNode[][];
   cursor: string | null;
   counts: LogLevelCounts;
   loading: boolean;
@@ -99,25 +99,29 @@ const reducer = (state: State, action: Action) => {
         ...node,
         clientsideKey: `csk${node.timestamp}-${ii}`,
       }));
-      const nodes = [...state.nodes, ...queuedNodes];
+
+      const copy = state.nodeChunks.slice();
+      copy.push(queuedNodes);
+
       const counts = {...state.counts};
       queuedNodes.forEach((node) => {
         const level = logNodeLevel(node);
         counts[level]++;
       });
-      return {nodes, counts, loading: action.hasMore, cursor: action.cursor};
+
+      return {nodeChunks: copy, counts, loading: action.hasMore, cursor: action.cursor};
     }
     case 'set-cursor':
       return {...state, cursor: action.cursor};
     case 'reset':
-      return {nodes: [], counts: emptyCounts, cursor: null, loading: true};
+      return {nodeChunks: [], counts: emptyCounts, cursor: null, loading: true};
     default:
       return state;
   }
 };
 
 const initialState: State = {
-  nodes: [],
+  nodeChunks: [] as LogNode[][],
   counts: emptyCounts,
   cursor: null,
   loading: true,
@@ -184,7 +188,7 @@ const useLogsProviderWithSubscription = (runId: string) => {
     }, BATCH_INTERVAL);
   }, [syncPipelineStatusToApolloCache]);
 
-  const {nodes, counts, cursor, loading} = state;
+  const {nodeChunks, counts, cursor, loading} = state;
 
   const {availability, disabled, status} = React.useContext(WebSocketContext);
   const lostWebsocket = !disabled && availability === 'available' && status === WebSocket.CLOSED;
@@ -227,10 +231,10 @@ const useLogsProviderWithSubscription = (runId: string) => {
 
   return React.useMemo(
     () =>
-      nodes !== null
-        ? {allNodes: nodes, counts, loading, subscriptionComponent}
-        : {allNodes: [], counts, loading, subscriptionComponent},
-    [counts, loading, nodes, subscriptionComponent],
+      nodeChunks !== null
+        ? {allNodeChunks: nodeChunks, counts, loading, subscriptionComponent}
+        : {allNodeChunks: [], counts, loading, subscriptionComponent},
+    [counts, loading, nodeChunks, subscriptionComponent],
   );
 };
 
@@ -285,7 +289,7 @@ const POLL_INTERVAL = 5000;
 const LogsProviderWithQuery = (props: LogsProviderWithQueryProps) => {
   const {children, runId} = props;
   const [state, dispatch] = React.useReducer(reducer, initialState);
-  const {counts, cursor, nodes} = state;
+  const {counts, cursor, nodeChunks} = state;
 
   const dependency = useTraceDependency('RunLogsQuery');
 
@@ -332,9 +336,9 @@ const LogsProviderWithQuery = (props: LogsProviderWithQueryProps) => {
   return (
     <>
       {children(
-        nodes !== null && nodes.length > 0
-          ? {allNodes: nodes, counts, loading: false}
-          : {allNodes: [], counts, loading: true},
+        nodeChunks !== null && nodeChunks.length > 0
+          ? {allNodeChunks: nodeChunks, counts, loading: false}
+          : {allNodeChunks: [], counts, loading: true},
       )}
     </>
   );
@@ -350,7 +354,7 @@ export const LogsProvider = (props: LogsProviderProps) => {
   }
 
   if (availability === 'attempting-to-connect') {
-    return <>{children({allNodes: [], counts: emptyCounts, loading: true})}</>;
+    return <>{children({allNodeChunks: [], counts: emptyCounts, loading: true})}</>;
   }
 
   return <LogsProviderWithSubscription runId={runId}>{children}</LogsProviderWithSubscription>;

js_modules/dagster-ui/packages/ui-core/src/runs/LogsScrollingTable.tsx

@@ -1,4 +1,4 @@
-import {Box, Colors, NonIdealState, Row} from '@dagster-io/ui-components';
+import {Box, Colors, NonIdealState, Row, SpinnerWithText} from '@dagster-io/ui-components';
 import {useVirtualizer} from '@tanstack/react-virtual';
 import {useEffect, useRef} from 'react';
 import styled from 'styled-components';
@@ -81,12 +81,10 @@ export const LogsScrollingTable = (props: Props) => {
   }, [totalHeight, virtualizer]);
 
   const content = () => {
-    if (logs.loading) {
+    if (logs.allNodeChunks.length === 0 && logs.loading) {
       return (
-        <Box margin={{top: 32}}>
-          <ListEmptyState>
-            <NonIdealState icon="spinner" title="Fetching logs..." />
-          </ListEmptyState>
+        <Box margin={{top: 32}} flex={{direction: 'column', alignItems: 'center'}}>
+          <SpinnerWithText label="Loading run logs…" />
         </Box>
       );
     }

js_modules/dagster-ui/packages/ui-core/src/runs/RunMetadataProvider.tsx

@@ -3,6 +3,7 @@ import * as React from 'react';
 import {LogsProviderLogs} from './LogsProvider';
 import {RunContext} from './RunContext';
 import {gql} from '../apollo-client';
+import {flattenNodeChunks} from './flattenNodeChunks';
 import {RunFragment} from './types/RunFragments.types';
 import {RunMetadataProviderMessageFragment} from './types/RunMetadataProvider.types';
 import {StepEventStatus} from '../graphql/types';
@@ -371,7 +372,8 @@ export const RunMetadataProvider = ({logs, children}: IRunMetadataProviderProps)
   const run = React.useContext(RunContext);
   const runMetadata = React.useMemo(() => extractMetadataFromRun(run), [run]);
   const metadata = React.useMemo(
-    () => (logs.loading ? runMetadata : extractMetadataFromLogs(logs.allNodes)),
+    () =>
+      logs.loading ? runMetadata : extractMetadataFromLogs(flattenNodeChunks(logs.allNodeChunks)),
     [logs, runMetadata],
   );
   return <>{children(metadata)}</>;

js_modules/dagster-ui/packages/ui-core/src/runs/StepLogsDialog.tsx

@@ -8,7 +8,7 @@ import {
   Mono,
   Spinner,
 } from '@dagster-io/ui-components';
-import {useState} from 'react';
+import {useMemo, useState} from 'react';
 import * as React from 'react';
 import {Link} from 'react-router-dom';
 import styled from 'styled-components';
@@ -20,6 +20,7 @@ import {LogsScrollingTable} from './LogsScrollingTable';
 import {LogType, LogsToolbar} from './LogsToolbar';
 import {IRunMetadataDict, RunMetadataProvider} from './RunMetadataProvider';
 import {titleForRun} from './RunUtils';
+import {flattenNodeChunks} from './flattenNodeChunks';
 import {useComputeLogFileKeyForSelection} from './useComputeLogFileKeyForSelection';
 import {DagsterEventType} from '../graphql/types';
 
@@ -113,9 +114,12 @@ export const StepLogsModalContent = ({
   const [logType, setComputeLogType] = useState<LogType>(LogType.structured);
   const [computeLogUrl, setComputeLogUrl] = React.useState<string | null>(null);
 
-  const firstLogForStep = logs.allNodes.find(
+  const flatLogs = useMemo(() => flattenNodeChunks(logs.allNodeChunks), [logs]);
+
+  const firstLogForStep = flatLogs.find(
     (l) => l.eventType === DagsterEventType.STEP_START && l.stepKey && stepKeys.includes(l.stepKey),
   );
+
   const firstLogForStepTime = firstLogForStep ? Number(firstLogForStep.timestamp) : 0;
 
   const [filter, setFilter] = useState<LogFilter>({

js_modules/dagster-ui/packages/ui-core/src/runs/filterLogs.tsx

@@ -1,11 +1,12 @@
 import {LogFilter, LogsProviderLogs} from './LogsProvider';
+import {flattenNodeChunks} from './flattenNodeChunks';
 import {eventTypeToDisplayType} from './getRunFilterProviders';
 import {logNodeLevel} from './logNodeLevel';
 import {LogNode} from './types';
 import {weakmapMemoize} from '../app/Util';
 
 export function filterLogs(logs: LogsProviderLogs, filter: LogFilter, filterStepKeys: string[]) {
-  const filteredNodes = logs.allNodes.filter((node) => {
+  const filteredNodes = flattenNodeChunks(logs.allNodeChunks).filter((node) => {
     // These events are used to determine which assets a run will materialize and are not intended
     // to be displayed in the Dagster UI. Pagination is offset based, so we remove these logs client-side.
     if (

js_modules/dagster-ui/packages/ui-core/src/runs/flattenNodeChunks.tsx

@@ -0,0 +1,6 @@
+import {LogNode} from './types';
+
+// https://jsbench.me/o8kqzo8olz/1
+export function flattenNodeChunks(nodeChunks: LogNode[][]) {
+  return ([] as LogNode[]).concat(...nodeChunks);
+}