From 4640552e5db1a8621003f0b1ad3711295ce3373c Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Wed, 5 Jun 2024 05:06:37 -0400 Subject: [PATCH 01/16] Split RootWorkspaceQuery by code location + move responsibility to WorkspaceProvider --- .../packages/ui-components/package.json | 2 +- .../dagster-ui/packages/ui-core/package.json | 2 +- .../packages/ui-core/src/assets/AssetView.tsx | 1 - .../AssetViewDefinition.fixtures.ts | 42 +- .../src/assets/__tests__/AssetTables.test.tsx | 7 +- .../src/assets/__tests__/AssetView.test.tsx | 2 +- ...LaunchAssetChoosePartitionsDialog.test.tsx | 7 +- .../LaunchAssetExecutionButton.test.tsx | 7 +- .../src/instance/DeploymentStatusProvider.tsx | 2 +- .../LeftNavRepositorySection.fixtures.tsx | 191 +++------ .../__fixtures__/useDaemonStatus.fixtures.tsx | 92 ++--- .../LeftNavRepositorySection.test.tsx | 30 +- .../nav/__tests__/useDaemonStatus.test.tsx | 50 +-- .../nav/types/useCodeLocationsStatus.types.ts | 20 - .../src/nav/useCodeLocationsStatus.tsx | 107 ++--- .../ui-core/src/overview/OverviewSensors.tsx | 9 +- .../__fixtures__/RunActionsMenu.fixtures.tsx | 59 +-- .../__fixtures__/RunsFilterInput.fixtures.tsx | 18 +- .../runs/__tests__/RunActionButtons.test.tsx | 2 +- .../runs/__tests__/RunActionsMenu.test.tsx | 4 +- .../runs/__tests__/RunFilterInput.test.tsx | 43 +- .../src/search/useIndexedDBCachedQuery.tsx | 203 +++++---- .../src/workspace/WorkspaceContext.tsx | 384 +++++++++--------- .../src/workspace/WorkspaceQueries.tsx | 107 +++++ .../__fixtures__/Workspace.fixtures.ts | 54 +++ .../workspace/types/WorkspaceQueries.types.ts | 117 ++++++ js_modules/dagster-ui/yarn.lock | 24 +- 27 files changed, 845 insertions(+), 741 deletions(-) delete mode 100644 js_modules/dagster-ui/packages/ui-core/src/nav/types/useCodeLocationsStatus.types.ts create mode 100644 js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceQueries.tsx create mode 100644 js_modules/dagster-ui/packages/ui-core/src/workspace/__fixtures__/Workspace.fixtures.ts create mode 100644 js_modules/dagster-ui/packages/ui-core/src/workspace/types/WorkspaceQueries.types.ts diff --git a/js_modules/dagster-ui/packages/ui-components/package.json b/js_modules/dagster-ui/packages/ui-components/package.json index 6a8e2e34e44ba..b94b524083825 100644 --- a/js_modules/dagster-ui/packages/ui-components/package.json +++ b/js_modules/dagster-ui/packages/ui-components/package.json @@ -66,7 +66,7 @@ "@storybook/react-webpack5": "^7.2.0", "@testing-library/dom": "^10.0.0", "@testing-library/jest-dom": "^6.4.2", - "@testing-library/react": "^15.0.3", + "@testing-library/react": "^16.0.0", "@testing-library/user-event": "^14.5.2", "@types/babel__core": "^7", "@types/babel__preset-env": "^7", diff --git a/js_modules/dagster-ui/packages/ui-core/package.json b/js_modules/dagster-ui/packages/ui-core/package.json index d4a10b3dcb97b..08e3d9b9dc9c4 100644 --- a/js_modules/dagster-ui/packages/ui-core/package.json +++ b/js_modules/dagster-ui/packages/ui-core/package.json @@ -107,7 +107,7 @@ "@storybook/react-webpack5": "^7.6.7", "@testing-library/dom": "^10.0.0", "@testing-library/jest-dom": "^6.4.2", - "@testing-library/react": "^15.0.3", + "@testing-library/react": "^16.0.0", "@testing-library/user-event": "^14.5.2", "@types/codemirror": "^5.60.5", "@types/color": "^3.0.2", diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx index f57f703fdd435..c3473539a211a 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx @@ -279,7 +279,6 @@ export const AssetView = ({assetKey, trace, headerBreadcrumbs}: Props) => { const codeSource = assetMetadata?.find((m) => isCanonicalCodeSourceEntry(m)) as | CodeReferencesMetadataEntry | undefined; - console.log(codeSource); return ( ({ - query: ROOT_WORKSPACE_QUERY, - variables: {}, - data: { - workspaceOrError: buildWorkspace({ - locationEntries: [ - buildWorkspaceLocationEntry({ - locationOrLoadError: buildRepositoryLocation({ - repositories: [ - buildRepository({ - id: '4d0b1967471d9a4682ccc97d12c1c508d0d9c2e1', - name: 'repo', - location: buildRepositoryLocation({ - id: 'test.py', - name: 'test.py', - }), - }), - ], +export const RootWorkspaceWithOneLocation = buildWorkspaceMocks([ + buildWorkspaceLocationEntry({ + locationOrLoadError: buildRepositoryLocation({ + repositories: [ + buildRepository({ + id: '4d0b1967471d9a4682ccc97d12c1c508d0d9c2e1', + name: 'repo', + location: buildRepositoryLocation({ + id: 'test.py', + name: 'test.py', }), }), ], }), - }, -}); + }), +]); diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/__tests__/AssetTables.test.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/__tests__/AssetTables.test.tsx index 96f06dd2888f4..99cf34cb1255d 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/__tests__/AssetTables.test.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/__tests__/AssetTables.test.tsx @@ -4,9 +4,8 @@ import userEvent from '@testing-library/user-event'; import {MemoryRouter} from 'react-router'; import {RecoilRoot} from 'recoil'; -import {buildWorkspace} from '../../graphql/types'; -import {buildWorkspaceContextMockedResponse} from '../../runs/__fixtures__/RunsFilterInput.fixtures'; import {WorkspaceProvider} from '../../workspace/WorkspaceContext'; +import {buildWorkspaceMocks} from '../../workspace/__fixtures__/Workspace.fixtures'; import {AssetsCatalogTable} from '../AssetsCatalogTable'; import { AssetCatalogGroupTableMock, @@ -17,7 +16,7 @@ import { SingleAssetQueryTrafficDashboard, } from '../__fixtures__/AssetTables.fixtures'; -const workspaceMock = buildWorkspaceContextMockedResponse(buildWorkspace({})); +const workspaceMocks = buildWorkspaceMocks([]); const MOCKS = [ AssetCatalogTableMock, @@ -26,7 +25,7 @@ const MOCKS = [ SingleAssetQueryMaterializedWithLatestRun, SingleAssetQueryMaterializedStaleAndLate, SingleAssetQueryLastRunFailed, - workspaceMock, + ...workspaceMocks, ]; // This file must be mocked because Jest can't handle `import.meta.url`. diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/__tests__/AssetView.test.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/__tests__/AssetView.test.tsx index 4617116d1c57f..b4a2eaa25ff1c 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/__tests__/AssetView.test.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/__tests__/AssetView.test.tsx @@ -59,7 +59,7 @@ describe('AssetView', () => { { it('Adding a dynamic partition when multiple assets selected', async () => { @@ -92,7 +91,7 @@ describe('launchAssetChoosePartitionsDialog', () => { assetASecondQueryMock, assetBSecondQueryMock, addPartitionMock, - workspaceMock, + ...workspaceMocks, ]} > ({})); @@ -653,7 +652,7 @@ function renderButton({ }), buildLaunchAssetLoaderMock([MULTI_ASSET_OUT_1.assetKey, MULTI_ASSET_OUT_2.assetKey]), buildLaunchAssetLoaderMock(assetKeys), - workspaceMock, + ...workspaceMocks, ...(launchMock ? [launchMock] : []), ]; diff --git a/js_modules/dagster-ui/packages/ui-core/src/instance/DeploymentStatusProvider.tsx b/js_modules/dagster-ui/packages/ui-core/src/instance/DeploymentStatusProvider.tsx index 2ad2305371ab9..1f65f1913d09d 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/instance/DeploymentStatusProvider.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/instance/DeploymentStatusProvider.tsx @@ -24,7 +24,7 @@ interface Props { export const DeploymentStatusProvider = (props: Props) => { const {children, include} = props; - const codeLocations = useCodeLocationsStatus(!include.has('code-locations')); + const codeLocations = useCodeLocationsStatus(); const daemons = useDaemonStatus(!include.has('daemons')); const value = React.useMemo(() => ({codeLocations, daemons}), [daemons, codeLocations]); diff --git a/js_modules/dagster-ui/packages/ui-core/src/nav/__fixtures__/LeftNavRepositorySection.fixtures.tsx b/js_modules/dagster-ui/packages/ui-core/src/nav/__fixtures__/LeftNavRepositorySection.fixtures.tsx index 506fb2014bb1b..34381877df7e0 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/nav/__fixtures__/LeftNavRepositorySection.fixtures.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/nav/__fixtures__/LeftNavRepositorySection.fixtures.tsx @@ -1,16 +1,13 @@ -import {MockedResponse} from '@apollo/client/testing'; - import { + WorkspaceLocationEntry, buildAssetGroup, buildPipeline, buildRepository, buildRepositoryLocation, - buildWorkspace, buildWorkspaceLocationEntry, } from '../../graphql/types'; -import {ROOT_WORKSPACE_QUERY} from '../../workspace/WorkspaceContext'; +import {buildWorkspaceMocks} from '../../workspace/__fixtures__/Workspace.fixtures'; import {DUNDER_REPO_NAME} from '../../workspace/buildRepoAddress'; -import {RootWorkspaceQuery} from '../../workspace/types/WorkspaceContext.types'; const buildRepo = ({ name, @@ -38,136 +35,66 @@ const buildRepo = ({ }); }; -export const buildWorkspaceQueryWithZeroLocations = (): MockedResponse => { - return { - request: { - query: ROOT_WORKSPACE_QUERY, - variables: {}, - }, - result: { - data: { - __typename: 'Query', - workspaceOrError: buildWorkspace({ - locationEntries: [], - }), - }, - }, - }; -}; +export const buildWorkspaceQueryWithZeroLocations = () => buildWorkspaceMocks([]); -export const buildWorkspaceQueryWithOneLocation = (): MockedResponse => { - return { - request: { - query: ROOT_WORKSPACE_QUERY, - variables: {}, - }, - result: { - data: { - __typename: 'Query', - workspaceOrError: buildWorkspace({ - locationEntries: [ - buildWorkspaceLocationEntry({ - id: 'ipsum-entry', - name: 'ipsum-entry', - locationOrLoadError: buildRepositoryLocation({ - id: 'ipsum', - name: 'ipsum', - repositories: [ - buildRepo({name: 'lorem', jobNames: ['my_pipeline', 'other_pipeline']}), - ], - }), - }), - ], +const locationEntries = [ + buildWorkspaceLocationEntry({ + id: 'ipsum-entry', + name: 'ipsum-entry', + locationOrLoadError: buildRepositoryLocation({ + id: 'ipsum', + name: 'ipsum', + repositories: [buildRepo({name: 'lorem', jobNames: ['my_pipeline', 'other_pipeline']})], + }), + }), + buildWorkspaceLocationEntry({ + id: 'bar-entry', + name: 'bar-entry', + locationOrLoadError: buildRepositoryLocation({ + id: 'bar', + name: 'bar', + repositories: [buildRepo({name: 'foo', jobNames: ['bar_job', 'd_job', 'e_job', 'f_job']})], + }), + }), + buildWorkspaceLocationEntry({ + id: 'abc_location-entry', + name: 'abc_location-entry', + locationOrLoadError: buildRepositoryLocation({ + id: 'abc_location', + name: 'abc_location', + repositories: [ + buildRepo({ + name: DUNDER_REPO_NAME, + jobNames: ['abc_job', 'def_job', 'ghi_job', 'jkl_job', 'mno_job', 'pqr_job'], }), - }, - }, - }; + ], + }), + }), +] as [WorkspaceLocationEntry, WorkspaceLocationEntry, WorkspaceLocationEntry]; + +export const buildWorkspaceQueryWithOneLocation = () => { + return buildWorkspaceMocks([locationEntries[0]]); }; -export const buildWorkspaceQueryWithThreeLocations = (): MockedResponse => { - return { - request: { - query: ROOT_WORKSPACE_QUERY, - variables: {}, - }, - result: { - data: { - __typename: 'Query', - workspaceOrError: buildWorkspace({ - locationEntries: [ - buildWorkspaceLocationEntry({ - id: 'ipsum-entry', - name: 'ipsum-entry', - locationOrLoadError: buildRepositoryLocation({ - id: 'ipsum', - name: 'ipsum', - repositories: [ - buildRepo({name: 'lorem', jobNames: ['my_pipeline', 'other_pipeline']}), - ], - }), - }), - buildWorkspaceLocationEntry({ - id: 'bar-entry', - name: 'bar-entry', - locationOrLoadError: buildRepositoryLocation({ - id: 'bar', - name: 'bar', - repositories: [ - buildRepo({name: 'foo', jobNames: ['bar_job', 'd_job', 'e_job', 'f_job']}), - ], - }), - }), - buildWorkspaceLocationEntry({ - id: 'abc_location-entry', - name: 'abc_location-entry', - locationOrLoadError: buildRepositoryLocation({ - id: 'abc_location', - name: 'abc_location', - repositories: [ - buildRepo({ - name: DUNDER_REPO_NAME, - jobNames: ['abc_job', 'def_job', 'ghi_job', 'jkl_job', 'mno_job', 'pqr_job'], - }), - ], - }), - }), - ], - }), - }, - }, - }; +export const buildWorkspaceQueryWithThreeLocations = () => { + return buildWorkspaceMocks(locationEntries); }; -export const buildWorkspaceQueryWithOneLocationAndAssetGroup = - (): MockedResponse => { - return { - request: { - query: ROOT_WORKSPACE_QUERY, - variables: {}, - }, - result: { - data: { - __typename: 'Query', - workspaceOrError: buildWorkspace({ - locationEntries: [ - buildWorkspaceLocationEntry({ - id: 'ipsum-entry', - name: 'ipsum-entry', - locationOrLoadError: buildRepositoryLocation({ - id: 'ipsum', - name: 'ipsum', - repositories: [ - buildRepo({ - name: 'lorem', - jobNames: ['my_pipeline', 'other_pipeline'], - assetGroupNames: ['my_asset_group'], - }), - ], - }), - }), - ], - }), - }, - }, - }; - }; +const entryWithOneLocationAndAssetGroup = buildWorkspaceLocationEntry({ + id: 'unique-entry', + name: 'unique-entry', + locationOrLoadError: buildRepositoryLocation({ + id: 'unique', + name: 'unique', + repositories: [ + buildRepo({ + name: 'entry', + jobNames: ['my_pipeline', 'other_pipeline'], + assetGroupNames: ['my_asset_group'], + }), + ], + }), +}); + +export const buildWorkspaceQueryWithOneLocationAndAssetGroup = () => + buildWorkspaceMocks([entryWithOneLocationAndAssetGroup]); diff --git a/js_modules/dagster-ui/packages/ui-core/src/nav/__fixtures__/useDaemonStatus.fixtures.tsx b/js_modules/dagster-ui/packages/ui-core/src/nav/__fixtures__/useDaemonStatus.fixtures.tsx index 37fb0d1bbde44..daabb6cd721c0 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/nav/__fixtures__/useDaemonStatus.fixtures.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/nav/__fixtures__/useDaemonStatus.fixtures.tsx @@ -12,13 +12,11 @@ import { buildRepositoryLocation, buildSchedule, buildSensor, - buildWorkspace, buildWorkspaceLocationEntry, } from '../../graphql/types'; import {InstanceWarningQuery} from '../../instance/types/useDaemonStatus.types'; import {INSTANCE_WARNING_QUERY} from '../../instance/useDaemonStatus'; -import {ROOT_WORKSPACE_QUERY} from '../../workspace/WorkspaceContext'; -import {RootWorkspaceQuery} from '../../workspace/types/WorkspaceContext.types'; +import {buildWorkspaceMocks} from '../../workspace/__fixtures__/Workspace.fixtures'; const buildRepo = ({ name, @@ -55,33 +53,18 @@ const buildRepo = ({ }); }; -export const buildWorkspaceQueryWithNoSchedulesOrSensors = - (): MockedResponse => { - return { - request: { - query: ROOT_WORKSPACE_QUERY, - variables: {}, - }, - result: { - data: { - __typename: 'Query', - workspaceOrError: buildWorkspace({ - locationEntries: [ - buildWorkspaceLocationEntry({ - id: 'ipsum-entry', - name: 'ipsum-entry', - locationOrLoadError: buildRepositoryLocation({ - id: 'ipsum', - name: 'ipsum', - repositories: [buildRepo({name: 'lorem'})], - }), - }), - ], - }), - }, - }, - }; - }; +export const buildWorkspaceQueryWithNoSchedulesOrSensors = () => + buildWorkspaceMocks([ + buildWorkspaceLocationEntry({ + id: 'ipsum-entry', + name: 'ipsum-entry', + locationOrLoadError: buildRepositoryLocation({ + id: 'ipsum', + name: 'ipsum', + repositories: [buildRepo({name: 'lorem'})], + }), + }), + ]); export const buildWorkspaceQueryWithScheduleAndSensor = ({ schedule, @@ -89,37 +72,24 @@ export const buildWorkspaceQueryWithScheduleAndSensor = ({ }: { schedule: InstigationStatus; sensor: InstigationStatus; -}): MockedResponse => { - return { - request: { - query: ROOT_WORKSPACE_QUERY, - variables: {}, - }, - result: { - data: { - __typename: 'Query', - workspaceOrError: buildWorkspace({ - locationEntries: [ - buildWorkspaceLocationEntry({ - id: 'ipsum-entry', - name: 'ipsum-entry', - locationOrLoadError: buildRepositoryLocation({ - id: 'ipsum', - name: 'ipsum', - repositories: [ - buildRepo({ - name: 'lorem', - schedules: {'my-schedule': schedule}, - sensors: {'my-sensor': sensor}, - }), - ], - }), - }), - ], - }), - }, - }, - }; +}) => { + return buildWorkspaceMocks([ + buildWorkspaceLocationEntry({ + id: 'ipsum-entry' + Math.random(), + name: 'ipsum-entry' + Math.random(), + locationOrLoadError: buildRepositoryLocation({ + id: 'ipsum', + name: 'ipsum', + repositories: [ + buildRepo({ + name: 'lorem', + schedules: {'my-schedule': schedule}, + sensors: {'my-sensor': sensor}, + }), + ], + }), + }), + ]); }; type DaemonHealth = {daemonType: string; healthy: boolean; required: boolean}[]; diff --git a/js_modules/dagster-ui/packages/ui-core/src/nav/__tests__/LeftNavRepositorySection.test.tsx b/js_modules/dagster-ui/packages/ui-core/src/nav/__tests__/LeftNavRepositorySection.test.tsx index 39bd0ab452211..cc1f3f43f7dda 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/nav/__tests__/LeftNavRepositorySection.test.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/nav/__tests__/LeftNavRepositorySection.test.tsx @@ -43,13 +43,14 @@ describe('Repository options', () => { afterEach(() => { window.localStorage.clear(); __resetForJest(); + jest.resetModules(); }); it('Correctly displays the current repository state', async () => { await act(() => render( - + @@ -71,7 +72,7 @@ describe('Repository options', () => { await act(() => render( - + @@ -94,7 +95,7 @@ describe('Repository options', () => { await act(() => render( - + @@ -116,7 +117,7 @@ describe('Repository options', () => { await act(() => render( - + @@ -151,7 +152,7 @@ describe('Repository options', () => { await act(() => render( - + @@ -175,7 +176,7 @@ describe('Repository options', () => { await act(() => render( - + @@ -214,7 +215,7 @@ describe('Repository options', () => { act(() => { render( - + @@ -247,8 +248,8 @@ describe('Repository options', () => { }; const mocks = [ - buildWorkspaceQueryWithZeroLocations(), - buildWorkspaceQueryWithThreeLocations(), + ...buildWorkspaceQueryWithZeroLocations(), + ...buildWorkspaceQueryWithThreeLocations(), ]; await act(() => @@ -269,7 +270,7 @@ describe('Repository options', () => { const reloadButton = screen.getByRole('button', {name: /refetch workspace/i}); await userEvent.click(reloadButton); - const loremHeader = await screen.findByRole('button', {name: /lorem/i}); + const loremHeader = await waitFor(() => screen.findByRole('button', {name: /lorem/i})); await waitFor(() => { expect(loremHeader).toBeVisible(); }); @@ -303,7 +304,10 @@ describe('Repository options', () => { ); }; - const mocks = [buildWorkspaceQueryWithOneLocation(), buildWorkspaceQueryWithZeroLocations()]; + const mocks = [ + ...buildWorkspaceQueryWithOneLocation(), + ...buildWorkspaceQueryWithZeroLocations(), + ]; await act(() => render( @@ -339,7 +343,7 @@ describe('Repository options', () => { await act(() => render( - + @@ -348,7 +352,7 @@ describe('Repository options', () => { ), ); - const repoHeader = await screen.findByRole('button', {name: /lorem/i}); + const repoHeader = await screen.findByRole('button', {name: /unique/i}); await userEvent.click(repoHeader); await waitFor(() => { diff --git a/js_modules/dagster-ui/packages/ui-core/src/nav/__tests__/useDaemonStatus.test.tsx b/js_modules/dagster-ui/packages/ui-core/src/nav/__tests__/useDaemonStatus.test.tsx index d93b970707fef..f706d2a38b9e8 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/nav/__tests__/useDaemonStatus.test.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/nav/__tests__/useDaemonStatus.test.tsx @@ -1,10 +1,10 @@ import {MockedProvider} from '@apollo/client/testing'; -import {render, screen} from '@testing-library/react'; -import {renderHook} from '@testing-library/react-hooks'; +import {render, renderHook, screen, waitFor} from '@testing-library/react'; import * as React from 'react'; import {InstigationStatus} from '../../graphql/types'; import {useDaemonStatus} from '../../instance/useDaemonStatus'; +import {__resetForJest} from '../../search/useIndexedDBCachedQuery'; import {WorkspaceProvider} from '../../workspace/WorkspaceContext'; import { buildInstanceWarningQuery, @@ -12,15 +12,18 @@ import { buildWorkspaceQueryWithScheduleAndSensor, } from '../__fixtures__/useDaemonStatus.fixtures'; +afterEach(() => { + __resetForJest(); +}); describe('useDaemonStatus', () => { describe('Scheduler daemon', () => { it('does not surface scheduler errors if there are no schedules', async () => { const daemonHealth = [{daemonType: 'SCHEDULER', healthy: false, required: true}]; - const {result, waitForNextUpdate} = renderHook(() => useDaemonStatus(), { + const {result} = renderHook(() => useDaemonStatus(), { wrapper: ({children}: {children: React.ReactNode}) => ( @@ -29,18 +32,17 @@ describe('useDaemonStatus', () => { ), }); - await waitForNextUpdate(); expect(result.current).toBeNull(); }); it('does not surface scheduler errors if there are no running schedules', async () => { const daemonHealth = [{daemonType: 'SCHEDULER', healthy: false, required: true}]; - const {result, waitForNextUpdate} = renderHook(() => useDaemonStatus(), { + const {result} = renderHook(() => useDaemonStatus(), { wrapper: ({children}: {children: React.ReactNode}) => ( { ), }); - await waitForNextUpdate(); expect(result.current).toBeNull(); }); it('does surface scheduler errors if there is a running schedule', async () => { + (window as any).__debug = true; const daemonHealth = [{daemonType: 'SCHEDULER', healthy: false, required: true}]; - const {result, waitFor} = renderHook(() => useDaemonStatus(), { + const {result} = renderHook(() => useDaemonStatus(), { wrapper: ({children}: {children: React.ReactNode}) => ( { render(
{result.current?.content}
); expect(screen.getByText(/1 daemon not running/i)).toBeVisible(); }); + (window as any).__debug = false; }); }); @@ -87,11 +90,11 @@ describe('useDaemonStatus', () => { it('does not surface sensor daemon errors if there are no sensors', async () => { const daemonHealth = [{daemonType: 'SENSOR', healthy: false, required: true}]; - const {result, waitForNextUpdate} = renderHook(() => useDaemonStatus(), { + const {result} = renderHook(() => useDaemonStatus(), { wrapper: ({children}: {children: React.ReactNode}) => ( @@ -100,18 +103,17 @@ describe('useDaemonStatus', () => { ), }); - await waitForNextUpdate(); expect(result.current).toBeNull(); }); it('does not surface sensor daemon errors if there are no running sensors', async () => { const daemonHealth = [{daemonType: 'SENSOR', healthy: false, required: true}]; - const {result, waitForNextUpdate} = renderHook(() => useDaemonStatus(), { + const {result} = renderHook(() => useDaemonStatus(), { wrapper: ({children}: {children: React.ReactNode}) => ( { ), }); - await waitForNextUpdate(); expect(result.current).toBeNull(); }); it('does surface sensor daemon errors if there is a running sensor', async () => { const daemonHealth = [{daemonType: 'SENSOR', healthy: false, required: true}]; - const {result, waitFor} = renderHook(() => useDaemonStatus(), { + const {result} = renderHook(() => useDaemonStatus(), { wrapper: ({children}: {children: React.ReactNode}) => ( { it('does not surface backfill daemon errors if there are no backfills', async () => { const daemonHealth = [{daemonType: 'BACKFILL', healthy: false, required: true}]; - const {result, waitForNextUpdate} = renderHook(() => useDaemonStatus(), { + const {result} = renderHook(() => useDaemonStatus(), { wrapper: ({children}: {children: React.ReactNode}) => ( @@ -171,18 +172,17 @@ describe('useDaemonStatus', () => { ), }); - await waitForNextUpdate(); expect(result.current).toBeNull(); }); it('does surface backfill daemon errors if there is a backfill', async () => { const daemonHealth = [{daemonType: 'BACKFILL', healthy: false, required: true}]; - const {result, waitFor} = renderHook(() => useDaemonStatus(), { + const {result} = renderHook(() => useDaemonStatus(), { wrapper: ({children}: {children: React.ReactNode}) => ( @@ -207,11 +207,11 @@ describe('useDaemonStatus', () => { {daemonType: 'BACKFILL', healthy: false, required: true}, ]; - const {result, waitFor} = renderHook(() => useDaemonStatus(), { + const {result} = renderHook(() => useDaemonStatus(), { wrapper: ({children}: {children: React.ReactNode}) => ( ; - -export type CodeLocationStatusQuery = { - __typename: 'Query'; - locationStatusesOrError: - | {__typename: 'PythonError'} - | { - __typename: 'WorkspaceLocationStatusEntries'; - entries: Array<{ - __typename: 'WorkspaceLocationStatusEntry'; - id: string; - name: string; - loadStatus: Types.RepositoryLocationLoadStatus; - }>; - }; -}; diff --git a/js_modules/dagster-ui/packages/ui-core/src/nav/useCodeLocationsStatus.tsx b/js_modules/dagster-ui/packages/ui-core/src/nav/useCodeLocationsStatus.tsx index 907fd21f22570..2f276e0fc95e3 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/nav/useCodeLocationsStatus.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/nav/useCodeLocationsStatus.tsx @@ -1,18 +1,14 @@ -import {gql, useQuery} from '@apollo/client'; import {Box, ButtonLink, Colors} from '@dagster-io/ui-components'; -import {useCallback, useContext, useState} from 'react'; +import {useCallback, useContext, useEffect, useLayoutEffect, useState} from 'react'; import {useHistory} from 'react-router-dom'; +import {atom, useRecoilValue} from 'recoil'; import styled from 'styled-components'; -import { - CodeLocationStatusQuery, - CodeLocationStatusQueryVariables, -} from './types/useCodeLocationsStatus.types'; import {showSharedToaster} from '../app/DomUtils'; -import {useQueryRefreshAtInterval} from '../app/QueryRefresh'; import {RepositoryLocationLoadStatus} from '../graphql/types'; import {StatusAndMessage} from '../instance/DeploymentStatusType'; import {WorkspaceContext} from '../workspace/WorkspaceContext'; +import {CodeLocationStatusQuery} from '../workspace/types/WorkspaceQueries.types'; type LocationStatusEntry = { loadStatus: RepositoryLocationLoadStatus; @@ -20,12 +16,15 @@ type LocationStatusEntry = { name: string; }; -const POLL_INTERVAL = 5 * 1000; - type EntriesById = Record; -export const useCodeLocationsStatus = (skip = false): StatusAndMessage | null => { - const {locationEntries, refetch} = useContext(WorkspaceContext); +export const codeLocationStatusAtom = atom({ + key: 'codeLocationStatusQuery', + default: undefined, +}); + +export const useCodeLocationsStatus = (): StatusAndMessage | null => { + const {locationEntries, data} = useContext(WorkspaceContext); const [previousEntriesById, setPreviousEntriesById] = useState(null); const history = useHistory(); @@ -37,28 +36,20 @@ export const useCodeLocationsStatus = (skip = false): StatusAndMessage | null => }, [history]); // Reload the workspace, but don't toast about it. - const reloadWorkspaceQuietly = useCallback(async () => { - setShowSpinner(true); - await refetch(); - setShowSpinner(false); - }, [refetch]); // Reload the workspace, and show a success or error toast upon completion. - const reloadWorkspaceLoudly = useCallback(async () => { - setShowSpinner(true); - const result = await refetch(); - setShowSpinner(false); - - const anyErrors = - result.data.workspaceOrError.__typename === 'PythonError' || - result.data.workspaceOrError.locationEntries.some( - (entry) => entry.locationOrLoadError?.__typename === 'PythonError', - ); + useEffect(() => { + const isFreshPageload = previousEntriesById === null; + const anyErrors = Object.values(data).some( + (entry) => + entry.__typename === 'PythonError' || + entry.locationOrLoadError?.__typename === 'PythonError', + ); const showViewButton = !alreadyViewingCodeLocations(); if (anyErrors) { - await showSharedToaster({ + showSharedToaster({ intent: 'warning', message: ( @@ -68,8 +59,8 @@ export const useCodeLocationsStatus = (skip = false): StatusAndMessage | null => ), icon: 'check_circle', }); - } else { - await showSharedToaster({ + } else if (!isFreshPageload) { + showSharedToaster({ intent: 'success', message: ( @@ -80,16 +71,20 @@ export const useCodeLocationsStatus = (skip = false): StatusAndMessage | null => icon: 'check_circle', }); } - }, [onClickViewButton, refetch]); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [data, onClickViewButton]); + + const codeLocationStatusQueryData = useRecoilValue(codeLocationStatusAtom); - const onLocationUpdate = async (data: CodeLocationStatusQuery) => { + useLayoutEffect(() => { const isFreshPageload = previousEntriesById === null; // Given the previous and current code locations, determine whether to show a) a loading spinner // and/or b) a toast indicating that a code location is being reloaded. const entries = - data?.locationStatusesOrError?.__typename === 'WorkspaceLocationStatusEntries' - ? data?.locationStatusesOrError.entries + codeLocationStatusQueryData?.locationStatusesOrError?.__typename === + 'WorkspaceLocationStatusEntries' + ? codeLocationStatusQueryData?.locationStatusesOrError.entries : []; let hasUpdatedEntries = entries.length !== Object.keys(previousEntriesById || {}).length; @@ -132,7 +127,6 @@ export const useCodeLocationsStatus = (skip = false): StatusAndMessage | null => // At least one code location has been removed. Reload, but don't make a big deal about it // since this was probably done manually. if (previousEntries.length > currentEntries.length) { - reloadWorkspaceQuietly(); return; } @@ -168,7 +162,7 @@ export const useCodeLocationsStatus = (skip = false): StatusAndMessage | null => return {addedEntries.length} code locations added; }; - await showSharedToaster({ + showSharedToaster({ intent: 'primary', message: ( @@ -179,7 +173,6 @@ export const useCodeLocationsStatus = (skip = false): StatusAndMessage | null => icon: 'add_circle', }); - reloadWorkspaceLoudly(); return; } @@ -192,7 +185,7 @@ export const useCodeLocationsStatus = (skip = false): StatusAndMessage | null => if (!anyPreviouslyLoading && anyCurrentlyLoading) { setShowSpinner(true); - await showSharedToaster({ + showSharedToaster({ intent: 'primary', message: ( @@ -211,30 +204,8 @@ export const useCodeLocationsStatus = (skip = false): StatusAndMessage | null => return; } - - // A location was previously loading, and no longer is. Our workspace is ready. Refetch it. - if (anyPreviouslyLoading && !anyCurrentlyLoading) { - reloadWorkspaceLoudly(); - return; - } - - if (hasUpdatedEntries) { - reloadWorkspaceLoudly(); - return; - } - }; - - const queryData = useQuery( - CODE_LOCATION_STATUS_QUERY, - { - fetchPolicy: 'network-only', - notifyOnNetworkStatusChange: true, - skip, - onCompleted: onLocationUpdate, - }, - ); - - useQueryRefreshAtInterval(queryData, POLL_INTERVAL); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [codeLocationStatusQueryData]); if (showSpinner) { return { @@ -274,17 +245,3 @@ const ViewCodeLocationsButton = ({onClick}: {onClick: () => void}) => { const ViewButton = styled(ButtonLink)` white-space: nowrap; `; - -const CODE_LOCATION_STATUS_QUERY = gql` - query CodeLocationStatusQuery { - locationStatusesOrError { - ... on WorkspaceLocationStatusEntries { - entries { - id - name - loadStatus - } - } - } - } -`; diff --git a/js_modules/dagster-ui/packages/ui-core/src/overview/OverviewSensors.tsx b/js_modules/dagster-ui/packages/ui-core/src/overview/OverviewSensors.tsx index a7b4f3c51bd43..fea23ea18a732 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/overview/OverviewSensors.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/overview/OverviewSensors.tsx @@ -103,12 +103,7 @@ export const OverviewSensors = () => { notifyOnNetworkStatusChange: true, }, ); - const {data: currentData, loading} = queryResultOverview; - const data = - currentData ?? - (cachedData?.workspaceOrError.__typename === 'Workspace' - ? (cachedData as Extract) - : null); + const {data, loading} = queryResultOverview; useBlockTraceOnQueryResult(queryResultOverview, 'OverviewSensorsQuery'); @@ -340,7 +335,7 @@ export const OverviewSensors = () => { ) : ( <> diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/__fixtures__/RunActionsMenu.fixtures.tsx b/js_modules/dagster-ui/packages/ui-core/src/runs/__fixtures__/RunActionsMenu.fixtures.tsx index e0c89d4cccf42..1ba3297465285 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/__fixtures__/RunActionsMenu.fixtures.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/__fixtures__/RunActionsMenu.fixtures.tsx @@ -7,11 +7,9 @@ import { buildRepositoryLocation, buildRepositoryOrigin, buildRun, - buildWorkspace, buildWorkspaceLocationEntry, } from '../../graphql/types'; -import {ROOT_WORKSPACE_QUERY} from '../../workspace/WorkspaceContext'; -import {RootWorkspaceQuery} from '../../workspace/types/WorkspaceContext.types'; +import {buildWorkspaceMocks} from '../../workspace/__fixtures__/Workspace.fixtures'; import {PIPELINE_ENVIRONMENT_QUERY} from '../RunActionsMenu'; import { PipelineEnvironmentQuery, @@ -57,41 +55,28 @@ export const buildRunActionsMenuFragment = ({hasReExecutePermission}: RunConfigI }); }; -export const buildMockRootWorkspaceQuery = (): MockedResponse => { - return { - request: { - query: ROOT_WORKSPACE_QUERY, - }, - result: { - data: { - __typename: 'Query', - workspaceOrError: buildWorkspace({ - id: 'workspace', - locationEntries: [ - buildWorkspaceLocationEntry({ - id: LOCATION_NAME, - locationOrLoadError: buildRepositoryLocation({ - id: LOCATION_NAME, - repositories: [ - buildRepository({ - id: REPO_NAME, - name: REPO_NAME, - pipelines: [ - buildPipeline({ - id: JOB_NAME, - name: JOB_NAME, - pipelineSnapshotId: SNAPSHOT_ID, - }), - ], - }), - ], +export const buildMockRootWorkspaceQuery = () => { + return buildWorkspaceMocks([ + buildWorkspaceLocationEntry({ + id: LOCATION_NAME, + locationOrLoadError: buildRepositoryLocation({ + id: LOCATION_NAME, + repositories: [ + buildRepository({ + id: REPO_NAME, + name: REPO_NAME, + pipelines: [ + buildPipeline({ + id: JOB_NAME, + name: JOB_NAME, + pipelineSnapshotId: SNAPSHOT_ID, }), - }), - ], - }), - }, - }, - }; + ], + }), + ], + }), + }), + ]); }; export const buildPipelineEnvironmentQuery = ( diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/__fixtures__/RunsFilterInput.fixtures.tsx b/js_modules/dagster-ui/packages/ui-core/src/runs/__fixtures__/RunsFilterInput.fixtures.tsx index c9691a984b294..86ef39f9f01d8 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/__fixtures__/RunsFilterInput.fixtures.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/__fixtures__/RunsFilterInput.fixtures.tsx @@ -1,26 +1,10 @@ import {MockedResponse} from '@apollo/client/testing'; -import {WorkspaceOrError, buildPipelineTagAndValues, buildRunTags} from '../../graphql/types'; -import {ROOT_WORKSPACE_QUERY} from '../../workspace/WorkspaceContext'; -import {RootWorkspaceQuery} from '../../workspace/types/WorkspaceContext.types'; +import {buildPipelineTagAndValues, buildRunTags} from '../../graphql/types'; import {DagsterTag} from '../RunTag'; import {RUN_TAG_VALUES_QUERY} from '../RunsFilterInput'; import {RunTagValuesQuery} from '../types/RunsFilterInput.types'; -export const buildWorkspaceContextMockedResponse = ( - workspaceOrError: WorkspaceOrError, -): MockedResponse => ({ - request: { - query: ROOT_WORKSPACE_QUERY, - }, - result: { - data: { - __typename: 'Query', - workspaceOrError, - }, - }, -}); - export function buildRunTagValuesQueryMockedResponse( tagKey: DagsterTag, values: string[], diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/__tests__/RunActionButtons.test.tsx b/js_modules/dagster-ui/packages/ui-core/src/runs/__tests__/RunActionButtons.test.tsx index f70825b651d47..ae9a8752ee3fc 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/__tests__/RunActionButtons.test.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/__tests__/RunActionButtons.test.tsx @@ -26,7 +26,7 @@ describe('RunActionButtons', () => { const Test = ({run}: {run: RunPageFragment}) => { return ( - + diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/__tests__/RunActionsMenu.test.tsx b/js_modules/dagster-ui/packages/ui-core/src/runs/__tests__/RunActionsMenu.test.tsx index 48e0c83b49c39..b171bc746a8fa 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/__tests__/RunActionsMenu.test.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/__tests__/RunActionsMenu.test.tsx @@ -17,7 +17,7 @@ describe('RunActionsMenu', () => { render( @@ -45,7 +45,7 @@ describe('RunActionsMenu', () => { render( diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/__tests__/RunFilterInput.test.tsx b/js_modules/dagster-ui/packages/ui-core/src/runs/__tests__/RunFilterInput.test.tsx index 49124ba600e2c..6758f47510328 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/__tests__/RunFilterInput.test.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/__tests__/RunFilterInput.test.tsx @@ -9,11 +9,11 @@ import { buildRepository, buildRepositoryLocation, buildRunTagKeys, - buildWorkspace, buildWorkspaceLocationEntry, } from '../../graphql/types'; import {calculateTimeRanges} from '../../ui/Filters/useTimeRangeFilter'; import {WorkspaceProvider} from '../../workspace/WorkspaceContext'; +import {buildWorkspaceMocks} from '../../workspace/__fixtures__/Workspace.fixtures'; import {DagsterTag} from '../RunTag'; import { RUN_TAG_KEYS_QUERY, @@ -24,34 +24,27 @@ import { useRunsFilterInput, useTagDataFilterValues, } from '../RunsFilterInput'; -import { - buildRunTagValuesQueryMockedResponse, - buildWorkspaceContextMockedResponse, -} from '../__fixtures__/RunsFilterInput.fixtures'; +import {buildRunTagValuesQueryMockedResponse} from '../__fixtures__/RunsFilterInput.fixtures'; import {RunTagKeysQuery} from '../types/RunsFilterInput.types'; -const workspaceMock = buildWorkspaceContextMockedResponse( - buildWorkspace({ - locationEntries: [ - buildWorkspaceLocationEntry({ - name: 'some_workspace', - locationOrLoadError: buildRepositoryLocation({ - name: 'some_location', - repositories: [ - buildRepository({ - name: 'some_repo', - pipelines: [ - buildPipeline({ - name: 'some_job', - }), - ], +const workspaceMocks = buildWorkspaceMocks([ + buildWorkspaceLocationEntry({ + name: 'some_workspace', + locationOrLoadError: buildRepositoryLocation({ + name: 'some_location', + repositories: [ + buildRepository({ + name: 'some_repo', + pipelines: [ + buildPipeline({ + name: 'some_job', }), ], }), - }), - ], + ], + }), }), -); +]); const runTagKeysMock: MockedResponse = { request: { @@ -168,7 +161,7 @@ function TestRunsFilterInput({ ); } return ( - + @@ -219,7 +212,7 @@ describe('', () => { tokens={tokens} onChange={onChange} enabledFilters={['job']} - mocks={[workspaceMock]} + mocks={workspaceMocks} />, ); diff --git a/js_modules/dagster-ui/packages/ui-core/src/search/useIndexedDBCachedQuery.tsx b/js_modules/dagster-ui/packages/ui-core/src/search/useIndexedDBCachedQuery.tsx index eaab4a86bb66d..914e688edd950 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/search/useIndexedDBCachedQuery.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/search/useIndexedDBCachedQuery.tsx @@ -1,6 +1,7 @@ -import {ApolloQueryResult, DocumentNode, OperationVariables, useApolloClient} from '@apollo/client'; +import {ApolloClient, DocumentNode, OperationVariables, useApolloClient} from '@apollo/client'; import {cache} from 'idb-lru-cache'; -import React from 'react'; +import memoize from 'lodash/memoize'; +import React, {useCallback} from 'react'; type CacheData = { data: TQuery; @@ -14,105 +15,139 @@ let fetchState: Record< } > = {}; -/** - * Returns data from the indexedDB cache initially while loading is true. - * Fetches data from the network/cache initially and does not receive any updates afterwards - * Uses fetch-policy: no-cache to avoid slow apollo cache normalization - */ +export class CacheManager { + private cache: ReturnType>>; + private key: string; + + constructor(key: string) { + this.key = `indexdbQueryCache:${key}`; + this.cache = cache>({dbName: this.key, maxCount: 1}); + } + + async get(version: number): Promise { + if (await this.cache.has('cache')) { + const {value} = await this.cache.get('cache'); + if (value && version === value.version) { + return value.data; + } + } + return null; + } + + set(data: TQuery, version: number): Promise { + return this.cache.set('cache', {data, version}, {expiry: new Date('3000-01-01')}); + } +} + +const getCacheManager = memoize((key: string) => { + return new CacheManager(key); +}); + +interface QueryHookParams { + key: string; + query: DocumentNode; + version: number; + variables?: TVariables; + onCompleted?: (data: TQuery) => void; +} + export function useIndexedDBCachedQuery({ key, query, version, variables, -}: { - key: string; - query: DocumentNode; - version: number; - variables?: TVariables; -}) { +}: QueryHookParams) { const client = useApolloClient(); - - const lru = React.useMemo( - () => cache>({dbName: `indexdbQueryCache:${key}`, maxCount: 1}), - [key], - ); - const [data, setData] = React.useState(null); + const [loading, setLoading] = React.useState(true); - const [loading, setLoading] = React.useState(false); - const [cacheBreaker, setCacheBreaker] = React.useState(0); - - React.useEffect(() => { - (async () => { - if (await lru.has('cache')) { - const {value} = await lru.get('cache'); - if (value) { - if (version === (value.version || null)) { - setData(value.data); - } - } - } - })(); - }, [lru, version, cacheBreaker]); - - const fetch = React.useCallback(async () => { - setLoading(true); - if (fetchState[key]) { - return await new Promise>((res) => { - fetchState[key]?.onFetched.push((value) => { - setCacheBreaker((v) => v + 1); - res(value); - }); + const fetch = useCallback( + async (bypassCache = false) => { + setLoading(true); + const newData = await getData({ + client, + key, + query, + variables, + version, + bypassCache, }); - } - fetchState[key] = {onFetched: []}; - // Use client.query here so that we initially use the apollo cache if any data is available in it - // and so that we don't subscribe to any updates to that cache (useLazyQuery and useQuery would both subscribe to updates to the - // cache which can be very slow) - const queryResult = await client.query({ - query, - variables, - fetchPolicy: 'no-cache', // Don't store the result in the cache, - // should help avoid page stuttering due to granular updates to the data - }); - const {data} = queryResult; - setLoading(false); - lru.set( - 'cache', - {data, version}, - { - expiry: new Date('3000-01-01'), // never expire, - }, - ); - delete fetchState[key]; - const onFetched = fetchState[key]?.onFetched; - try { - setData(data); - } catch (e) { - setTimeout(() => { - throw e; - }); - } - onFetched?.forEach((cb) => { - try { - cb(queryResult); - } catch (e) { - setTimeout(() => { - throw e; - }); - } - }); + setData(newData); + setLoading(false); + }, + [client, key, query, version, variables], + ); - return queryResult; - }, [client, key, lru, query, variables, version]); + React.useEffect(() => { + fetch(); + }, [fetch]); return { - fetch, data, loading, + fetch: useCallback(() => fetch(true), [fetch]), }; } +interface FetchParams { + client: ApolloClient; + key: string; + query: DocumentNode; + variables?: TVariables; + version: number; + bypassCache?: boolean; +} + +export async function getData({ + client, + key, + query, + variables, + version, + bypassCache = false, +}: FetchParams): Promise { + const cacheManager = getCacheManager(key); + + if (!bypassCache) { + const cachedData = await cacheManager.get(version); + if (cachedData !== null) { + return cachedData; + } + } + + const currentState = fetchState[key]; + // Handle concurrent fetch requests + if (currentState) { + return new Promise((resolve) => { + currentState!.onFetched.push(resolve as any); + }); + } + + const state = {onFetched: [] as ((value: any) => void)[]}; + fetchState[key] = state; + + const queryResult = await client.query({ + query, + variables, + fetchPolicy: 'no-cache', + }); + + const {data} = queryResult; + await cacheManager.set(data, version); + + const onFetchedHandlers = state.onFetched; + delete fetchState[key]; // Clean up fetch state after handling + + onFetchedHandlers.forEach((handler) => handler(data)); // Notify all waiting fetches + + return data; +} + +export async function getCachedData({key, version}: {key: string; version: number}) { + const cacheManager = getCacheManager(key); + return await cacheManager.get(version); +} + export const __resetForJest = () => { fetchState = {}; }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx index 22b029992896b..dd44d84b78524 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx @@ -1,30 +1,35 @@ -import {ApolloQueryResult, gql} from '@apollo/client'; +import {useApolloClient} from '@apollo/client'; import sortBy from 'lodash/sortBy'; -import * as React from 'react'; -import {useContext, useMemo} from 'react'; +import React, {useCallback, useContext, useEffect, useMemo, useRef, useState} from 'react'; -import {REPOSITORY_INFO_FRAGMENT} from './RepositoryInformation'; +import {CODE_LOCATION_STATUS_QUERY, LOCATION_WORKSPACE_QUERY} from './WorkspaceQueries'; import {buildRepoAddress} from './buildRepoAddress'; import {findRepoContainingPipeline} from './findRepoContainingPipeline'; import {RepoAddress} from './types'; import { - RootWorkspaceQuery, - RootWorkspaceQueryVariables, WorkspaceLocationFragment, WorkspaceLocationNodeFragment, WorkspaceRepositoryFragment, WorkspaceScheduleFragment, WorkspaceSensorFragment, } from './types/WorkspaceContext.types'; +import { + CodeLocationStatusQuery, + CodeLocationStatusQueryVariables, + LocationWorkspaceQuery, + LocationWorkspaceQueryVariables, +} from './types/WorkspaceQueries.types'; import {AppContext} from '../app/AppContext'; -import {PYTHON_ERROR_FRAGMENT} from '../app/PythonErrorFragment'; +import {useRefreshAtInterval} from '../app/QueryRefresh'; import {PythonErrorFragment} from '../app/types/PythonErrorFragment.types'; import {PipelineSelector} from '../graphql/types'; import {useStateWithStorage} from '../hooks/useStateWithStorage'; -import {BASIC_INSTIGATION_STATE_FRAGMENT} from '../overview/BasicInstigationStateFragment'; -import {useIndexedDBCachedQuery} from '../search/useIndexedDBCachedQuery'; -import {SENSOR_SWITCH_FRAGMENT} from '../sensors/SensorSwitch'; +import {useUpdatingRef} from '../hooks/useUpdatingRef'; +import {getCachedData, getData, useIndexedDBCachedQuery} from '../search/useIndexedDBCachedQuery'; +const CODE_LOCATION_STATUS_QUERY_KEY = 'CodeLocationStatusQuery'; +const CODE_LOCATION_STATUS_QUERY_VERSION = 3; +const LOCATION_WORKSPACE_QUERY_VERSION = 5; type Repository = WorkspaceRepositoryFragment; type RepositoryLocation = WorkspaceLocationFragment; @@ -40,14 +45,13 @@ export interface DagsterRepoOption { type SetVisibleOrHiddenFn = (repoAddresses: RepoAddress[]) => void; type WorkspaceState = { - error: PythonErrorFragment | null; loading: boolean; locationEntries: WorkspaceRepositoryLocationNode[]; allRepos: DagsterRepoOption[]; visibleRepos: DagsterRepoOption[]; - data: RootWorkspaceQuery | null; + data: Record; + refetch: () => Promise; - refetch: () => Promise>; toggleVisible: SetVisibleOrHiddenFn; setVisible: SetVisibleOrHiddenFn; setHidden: SetVisibleOrHiddenFn; @@ -59,170 +63,151 @@ export const WorkspaceContext = React.createContext( export const HIDDEN_REPO_KEYS = 'dagster.hidden-repo-keys'; -export const ROOT_WORKSPACE_QUERY = gql` - query RootWorkspaceQuery { - workspaceOrError { - ... on Workspace { - id - locationEntries { - id - ...WorkspaceLocationNode - } - } - ...PythonErrorFragment - } - } +export const WorkspaceProvider = ({children}: {children: React.ReactNode}) => { + const {localCacheIdPrefix} = useContext(AppContext); + const codeLocationStatusQueryResult = useIndexedDBCachedQuery< + CodeLocationStatusQuery, + CodeLocationStatusQueryVariables + >({ + query: CODE_LOCATION_STATUS_QUERY, + version: CODE_LOCATION_STATUS_QUERY_VERSION, + key: `${localCacheIdPrefix}/${CODE_LOCATION_STATUS_QUERY_KEY}`, + }); + const fetch = codeLocationStatusQueryResult.fetch; + useRefreshAtInterval({ + refresh: useCallback(async () => { + return await fetch(); + }, [fetch]), + intervalMs: 5000, + leading: true, + }); - fragment WorkspaceLocationNode on WorkspaceLocationEntry { - id - name - loadStatus - displayMetadata { - ...WorkspaceDisplayMetadata - } - updatedTimestamp - featureFlags { - name - enabled - } - locationOrLoadError { - ... on RepositoryLocation { - id - ...WorkspaceLocation - } - ...PythonErrorFragment - } - } + const {data} = codeLocationStatusQueryResult; - fragment WorkspaceDisplayMetadata on RepositoryMetadata { - key - value - } + const locations = useMemo(() => getLocations(data), [data]); + const prevLocations = useRef({}); - fragment WorkspaceLocation on RepositoryLocation { - id - isReloadSupported - serverId - name - dagsterLibraryVersions { - name - version - } - repositories { - id - ...WorkspaceRepository - } - } + const didInitiateFetchFromCache = useRef(false); + const [didLoadCachedData, setDidLoadCachedData] = useState(false); - fragment WorkspaceRepository on Repository { - id - name - pipelines { - id - name - isJob - isAssetJob - pipelineSnapshotId - } - schedules { - id - ...WorkspaceSchedule - } - sensors { - id - ...WorkspaceSensor - } - partitionSets { - id - mode - pipelineName - } - assetGroups { - id - groupName - } - allTopLevelResourceDetails { - id - name - } - ...RepositoryInfoFragment - } + const [locationsData, setLocationsData] = React.useState< + Record + >({}); - fragment WorkspaceSchedule on Schedule { - id - cronSchedule - executionTimezone - mode - name - pipelineName - scheduleState { - id - selectorId - status + useEffect(() => { + // Load data from the cache + if (didInitiateFetchFromCache.current) { + return; } - } + didInitiateFetchFromCache.current = true; + (async () => { + const data = await getCachedData({ + key: `${localCacheIdPrefix}/${CODE_LOCATION_STATUS_QUERY_KEY}`, + version: CODE_LOCATION_STATUS_QUERY_VERSION, + }); + const cachedLocations = getLocations(data); + const prevCachedLocations: typeof locations = {}; + + await Promise.all([ + ...Object.values(cachedLocations).map(async (location) => { + const locationData = await getCachedData({ + key: `${localCacheIdPrefix}/${locationWorkspaceKey(location.name)}`, + version: LOCATION_WORKSPACE_QUERY_VERSION, + }); + const entry = locationData?.workspaceLocationEntryOrError; + + if (!entry) { + return; + } + setLocationsData((locationsData) => + Object.assign({}, locationsData, { + [location.name]: entry, + }), + ); + + if (entry.__typename === 'WorkspaceLocationEntry') { + prevCachedLocations[location.name] = location; + } + }), + ]); + prevLocations.current = prevCachedLocations; + setDidLoadCachedData(true); + })(); + }, [localCacheIdPrefix, locations]); + + const client = useApolloClient(); + + const refetchLocation = useCallback( + async (name: string) => { + const locationData = await getData({ + client, + query: LOCATION_WORKSPACE_QUERY, + key: `${localCacheIdPrefix}/${locationWorkspaceKey(name)}`, + version: LOCATION_WORKSPACE_QUERY_VERSION, + variables: { + name, + }, + bypassCache: true, + }); + const entry = locationData?.workspaceLocationEntryOrError; + setLocationsData((locationsData) => + Object.assign({}, locationsData, { + [name]: entry, + }), + ); + return locationData; + }, + [client, localCacheIdPrefix], + ); - fragment WorkspaceSensor on Sensor { - id - jobOriginId - name - targets { - mode - pipelineName + const locationsToFetch = useMemo(() => { + if (!didLoadCachedData) { + return []; } - sensorState { - id - selectorId - status - ...BasicInstigationStateFragment + const toFetch = Object.values(locations).filter((loc) => { + const prev = prevLocations.current?.[loc.name]; + return prev?.updateTimestamp !== loc.updateTimestamp || prev?.loadStatus !== loc.loadStatus; + }); + prevLocations.current = locations; + return toFetch; + }, [locations, didLoadCachedData]); + + useEffect(() => { + locationsToFetch.forEach(async (location) => { + refetchLocation(location.name); + }); + }, [refetchLocation, locationsToFetch]); + + const locationsRemoved = useMemo(() => { + return Object.values(prevLocations.current).filter((loc) => !locations[loc.name]); + }, [locations]); + + useEffect(() => { + if (locationsRemoved.length) { + setLocationsData((locationsData) => { + const copy = {...locationsData}; + locationsRemoved.forEach((loc) => { + delete copy[loc.name]; + }); + return copy; + }); } - sensorType - ...SensorSwitchFragment - } - - ${PYTHON_ERROR_FRAGMENT} - ${REPOSITORY_INFO_FRAGMENT} - ${SENSOR_SWITCH_FRAGMENT} - ${BASIC_INSTIGATION_STATE_FRAGMENT} -`; - -/** - * A hook that supplies the current workspace state of Dagster UI, including the current - * "active" repo based on the URL or localStorage, all fetched repositories available - * in the workspace, and loading/error state for the relevant query. - */ -const useWorkspaceState = (): WorkspaceState => { - const {localCacheIdPrefix} = useContext(AppContext); - const { - data, - loading, - fetch: refetch, - } = useIndexedDBCachedQuery({ - query: ROOT_WORKSPACE_QUERY, - key: `${localCacheIdPrefix}/RootWorkspace`, - version: 1, - }); - useMemo(() => refetch(), [refetch]); - - const workspaceOrError = data?.workspaceOrError; - - const locationEntries = React.useMemo( - () => (workspaceOrError?.__typename === 'Workspace' ? workspaceOrError.locationEntries : []), - [workspaceOrError], + }, [locationsRemoved]); + + const locationEntries = useMemo( + () => + Object.values(locationsData).filter( + (entry): entry is WorkspaceLocationNodeFragment => + entry.__typename === 'WorkspaceLocationEntry', + ), + [locationsData], ); - const {allRepos, error} = React.useMemo(() => { + const {allRepos} = React.useMemo(() => { let allRepos: DagsterRepoOption[] = []; - if (!workspaceOrError) { - return {allRepos, error: null}; - } - - if (workspaceOrError.__typename === 'PythonError') { - return {allRepos, error: workspaceOrError}; - } allRepos = sortBy( - workspaceOrError.locationEntries.reduce((accum, locationEntry) => { + locationEntries.reduce((accum, locationEntry) => { if (locationEntry.locationOrLoadError?.__typename !== 'RepositoryLocation') { return accum; } @@ -237,25 +222,57 @@ const useWorkspaceState = (): WorkspaceState => { (r) => `${r.repositoryLocation.name}:${r.repository.name}`, ); - return {error: null, allRepos}; - }, [workspaceOrError]); + return {allRepos}; + }, [locationEntries]); const {visibleRepos, toggleVisible, setVisible, setHidden} = useVisibleRepos(allRepos); - return { - refetch, - loading: loading && !data, // Only "loading" on initial load. - error, - locationEntries, - data, - allRepos, - visibleRepos, - toggleVisible, - setVisible, - setHidden, - }; + const locationsRef = useUpdatingRef(locations); + + const refetch = useCallback(async () => { + return await Promise.all( + Object.values(locationsRef.current).map((location) => refetchLocation(location.name)), + ); + }, [locationsRef, refetchLocation]); + + return ( + + {children} + + ); }; +function getLocations(d: CodeLocationStatusQuery | undefined | null) { + const locations = + d?.locationStatusesOrError?.__typename === 'WorkspaceLocationStatusEntries' + ? d?.locationStatusesOrError.entries + : []; + return locations.reduce( + (accum, loc) => { + accum[loc.name] = loc; + return accum; + }, + {} as Record, + ); +} + +function locationWorkspaceKey(name: string) { + return `LocationWorkspace/${name}`; +} + /** * useVisibleRepos returns `{reposForKeys, toggleVisible, setVisible, setHidden}` and internally * mirrors the current selection into localStorage so that the default selection in new browser @@ -352,16 +369,9 @@ const useVisibleRepos = ( const getRepositoryOptionHash = (a: DagsterRepoOption) => `${a.repository.name}:${a.repositoryLocation.name}`; - -export const WorkspaceProvider = ({children}: {children: React.ReactNode}) => { - const workspaceState = useWorkspaceState(); - - return {children}; -}; - export const useRepositoryOptions = () => { - const {allRepos: options, loading, error} = React.useContext(WorkspaceContext); - return {options, loading, error}; + const {allRepos: options, loading} = React.useContext(WorkspaceContext); + return {options, loading}; }; export const useRepository = (repoAddress: RepoAddress | null) => { @@ -409,7 +419,7 @@ export const getFeatureFlagForCodeLocation = ( }; export const useFeatureFlagForCodeLocation = (locationName: string, flagName: string) => { - const {locationEntries} = useWorkspaceState(); + const {locationEntries} = useContext(WorkspaceContext); return getFeatureFlagForCodeLocation(locationEntries, locationName, flagName); }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceQueries.tsx b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceQueries.tsx new file mode 100644 index 0000000000000..05f5c8434b2b7 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceQueries.tsx @@ -0,0 +1,107 @@ +import {gql} from '@apollo/client'; + +import {REPOSITORY_INFO_FRAGMENT} from './RepositoryInformation'; +import {PYTHON_ERROR_FRAGMENT} from '../app/PythonErrorFragment'; + +export const LOCATION_WORKSPACE_QUERY = gql` + query LocationWorkspaceQuery($name: String!) { + workspaceLocationEntryOrError(name: $name) { + __typename + ... on WorkspaceLocationEntry { + id + name + loadStatus + displayMetadata { + key + value + } + updatedTimestamp + featureFlags { + name + enabled + } + locationOrLoadError { + ... on RepositoryLocation { + id + name + dagsterLibraryVersions { + name + version + } + repositories { + ...RepositoryInfoFragment + id + name + pipelines { + id + name + isJob + isAssetJob + pipelineSnapshotId + } + schedules { + id + name + cronSchedule + executionTimezone + mode + pipelineName + scheduleState { + id + selectorId + status + } + } + sensors { + id + name + jobOriginId + targets { + mode + pipelineName + } + sensorState { + id + selectorId + status + } + sensorType + } + partitionSets { + id + mode + pipelineName + } + assetGroups { + id + groupName + } + allTopLevelResourceDetails { + id + name + } + } + } + ...PythonErrorFragment + } + } + } + } + ${PYTHON_ERROR_FRAGMENT} + ${REPOSITORY_INFO_FRAGMENT} +`; + +export const CODE_LOCATION_STATUS_QUERY = gql` + query CodeLocationStatusQuery { + locationStatusesOrError { + ... on WorkspaceLocationStatusEntries { + entries { + id + name + loadStatus + updateTimestamp + } + } + } + } +`; diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/__fixtures__/Workspace.fixtures.ts b/js_modules/dagster-ui/packages/ui-core/src/workspace/__fixtures__/Workspace.fixtures.ts new file mode 100644 index 0000000000000..34a1ac7bfed25 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/__fixtures__/Workspace.fixtures.ts @@ -0,0 +1,54 @@ +import {MockedResponse} from '@apollo/client/testing'; + +import { + WorkspaceLocationEntry, + WorkspaceLocationStatusEntry, + buildWorkspaceLocationStatusEntries, + buildWorkspaceLocationStatusEntry, +} from '../../graphql/types'; +import {buildQueryMock} from '../../testing/mocking'; +import {CODE_LOCATION_STATUS_QUERY, LOCATION_WORKSPACE_QUERY} from '../WorkspaceQueries'; +import { + CodeLocationStatusQuery, + CodeLocationStatusQueryVariables, + LocationWorkspaceQuery, + LocationWorkspaceQueryVariables, +} from '../types/WorkspaceQueries.types'; + +export const buildCodeLocationsStatusQuery = ( + entries: WorkspaceLocationStatusEntry[], +): MockedResponse => { + return buildQueryMock({ + query: CODE_LOCATION_STATUS_QUERY, + variables: {}, + data: { + locationStatusesOrError: buildWorkspaceLocationStatusEntries({ + entries, + }), + }, + }); +}; + +export const buildWorkspaceMocks = (entries: WorkspaceLocationEntry[]) => { + return [ + ...entries.map((entry) => + buildQueryMock({ + query: LOCATION_WORKSPACE_QUERY, + variables: { + name: entry.name, + }, + data: { + workspaceLocationEntryOrError: entry, + }, + }), + ), + buildCodeLocationsStatusQuery( + entries.map((entry) => + buildWorkspaceLocationStatusEntry({ + ...entry, + __typename: 'WorkspaceLocationStatusEntry', + }), + ), + ), + ]; +}; diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/types/WorkspaceQueries.types.ts b/js_modules/dagster-ui/packages/ui-core/src/workspace/types/WorkspaceQueries.types.ts new file mode 100644 index 0000000000000..518bcf2d173e1 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/types/WorkspaceQueries.types.ts @@ -0,0 +1,117 @@ +// Generated GraphQL types, do not edit manually. + +import * as Types from '../../graphql/types'; + +export type LocationWorkspaceQueryVariables = Types.Exact<{ + name: Types.Scalars['String']['input']; +}>; + +export type LocationWorkspaceQuery = { + __typename: 'Query'; + workspaceLocationEntryOrError: + | {__typename: 'PythonError'} + | { + __typename: 'WorkspaceLocationEntry'; + id: string; + name: string; + loadStatus: Types.RepositoryLocationLoadStatus; + updatedTimestamp: number; + displayMetadata: Array<{__typename: 'RepositoryMetadata'; key: string; value: string}>; + featureFlags: Array<{__typename: 'FeatureFlag'; name: string; enabled: boolean}>; + locationOrLoadError: + | { + __typename: 'PythonError'; + message: string; + stack: Array; + errorChain: Array<{ + __typename: 'ErrorChainLink'; + isExplicitLink: boolean; + error: {__typename: 'PythonError'; message: string; stack: Array}; + }>; + } + | { + __typename: 'RepositoryLocation'; + id: string; + name: string; + dagsterLibraryVersions: Array<{ + __typename: 'DagsterLibraryVersion'; + name: string; + version: string; + }> | null; + repositories: Array<{ + __typename: 'Repository'; + id: string; + name: string; + pipelines: Array<{ + __typename: 'Pipeline'; + id: string; + name: string; + isJob: boolean; + isAssetJob: boolean; + pipelineSnapshotId: string; + }>; + schedules: Array<{ + __typename: 'Schedule'; + id: string; + name: string; + cronSchedule: string; + executionTimezone: string | null; + mode: string; + pipelineName: string; + scheduleState: { + __typename: 'InstigationState'; + id: string; + selectorId: string; + status: Types.InstigationStatus; + }; + }>; + sensors: Array<{ + __typename: 'Sensor'; + id: string; + name: string; + jobOriginId: string; + sensorType: Types.SensorType; + targets: Array<{__typename: 'Target'; mode: string; pipelineName: string}> | null; + sensorState: { + __typename: 'InstigationState'; + id: string; + selectorId: string; + status: Types.InstigationStatus; + }; + }>; + partitionSets: Array<{ + __typename: 'PartitionSet'; + id: string; + mode: string; + pipelineName: string; + }>; + assetGroups: Array<{__typename: 'AssetGroup'; id: string; groupName: string}>; + allTopLevelResourceDetails: Array<{ + __typename: 'ResourceDetails'; + id: string; + name: string; + }>; + }>; + } + | null; + } + | null; +}; + +export type CodeLocationStatusQueryVariables = Types.Exact<{[key: string]: never}>; + +export type CodeLocationStatusQuery = { + __typename: 'Query'; + locationStatusesOrError: + | {__typename: 'PythonError'} + | { + __typename: 'WorkspaceLocationStatusEntries'; + entries: Array<{ + __typename: 'WorkspaceLocationStatusEntry'; + id: string; + name: string; + loadStatus: Types.RepositoryLocationLoadStatus; + updateTimestamp: number; + }>; + }; +}; diff --git a/js_modules/dagster-ui/yarn.lock b/js_modules/dagster-ui/yarn.lock index b0e76f252ee33..67f55c477689c 100644 --- a/js_modules/dagster-ui/yarn.lock +++ b/js_modules/dagster-ui/yarn.lock @@ -2486,7 +2486,7 @@ __metadata: "@storybook/react-webpack5": "npm:^7.2.0" "@testing-library/dom": "npm:^10.0.0" "@testing-library/jest-dom": "npm:^6.4.2" - "@testing-library/react": "npm:^15.0.3" + "@testing-library/react": "npm:^16.0.0" "@testing-library/user-event": "npm:^14.5.2" "@types/babel__core": "npm:^7" "@types/babel__preset-env": "npm:^7" @@ -2576,7 +2576,7 @@ __metadata: "@tanstack/react-virtual": "npm:^3.0.1" "@testing-library/dom": "npm:^10.0.0" "@testing-library/jest-dom": "npm:^6.4.2" - "@testing-library/react": "npm:^15.0.3" + "@testing-library/react": "npm:^16.0.0" "@testing-library/react-hooks": "npm:^7.0.2" "@testing-library/user-event": "npm:^14.5.2" "@types/codemirror": "npm:^5.60.5" @@ -7335,17 +7335,23 @@ __metadata: languageName: node linkType: hard -"@testing-library/react@npm:^15.0.3": - version: 15.0.3 - resolution: "@testing-library/react@npm:15.0.3" +"@testing-library/react@npm:^16.0.0": + version: 16.0.0 + resolution: "@testing-library/react@npm:16.0.0" dependencies: "@babel/runtime": "npm:^7.12.5" - "@testing-library/dom": "npm:^10.0.0" - "@types/react-dom": "npm:^18.0.0" peerDependencies: + "@testing-library/dom": ^10.0.0 + "@types/react": ^18.0.0 + "@types/react-dom": ^18.0.0 react: ^18.0.0 react-dom: ^18.0.0 - checksum: 10/3e094accf49bdfba141ac53cad4902831fbef9d2d4a2c39a908820b141569aa8ca85e35f95560749ab46ad1db1be38a5382d38a0cb41a55a5d0b8afe116f2ba0 + peerDependenciesMeta: + "@types/react": + optional: true + "@types/react-dom": + optional: true + checksum: 10/b32894be94e31276138decfa6bcea69dfebc0c37cf91499ff6c878f41eb1154a43a7df6eb1e72e7bede78468af6cb67ca59e4acd3206b41f3ecdae2c6efdf67e languageName: node linkType: hard @@ -8006,7 +8012,7 @@ __metadata: languageName: node linkType: hard -"@types/react-dom@npm:>=16.9.0, @types/react-dom@npm:^18.0.0, @types/react-dom@npm:^18.2.0": +"@types/react-dom@npm:>=16.9.0, @types/react-dom@npm:^18.2.0": version: 18.2.7 resolution: "@types/react-dom@npm:18.2.7" dependencies: From bfd1d699fe9ab30bd4a8ba00f19c3ca16b35484f Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Wed, 5 Jun 2024 05:36:19 -0400 Subject: [PATCH 02/16] types --- .../src/instance/flattenCodeLocationRows.tsx | 2 +- .../ui-core/src/overview/OverviewSensors.tsx | 10 +- .../src/workspace/CodeLocationRowSet.tsx | 2 +- .../workspace/VirtualizedCodeLocationRow.tsx | 2 +- .../src/workspace/WorkspaceContext.tsx | 2 +- .../src/workspace/WorkspaceQueries.tsx | 171 ++++---- .../getFeatureFlagForCodeLocation.test.tsx | 2 +- .../workspace/types/WorkspaceContext.types.ts | 396 ------------------ .../workspace/types/WorkspaceQueries.types.ts | 221 +++++++++- 9 files changed, 309 insertions(+), 499 deletions(-) delete mode 100644 js_modules/dagster-ui/packages/ui-core/src/workspace/types/WorkspaceContext.types.ts diff --git a/js_modules/dagster-ui/packages/ui-core/src/instance/flattenCodeLocationRows.tsx b/js_modules/dagster-ui/packages/ui-core/src/instance/flattenCodeLocationRows.tsx index e316e1a5701d0..b9d7513e2dd71 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/instance/flattenCodeLocationRows.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/instance/flattenCodeLocationRows.tsx @@ -1,5 +1,5 @@ import {CodeLocationRowType} from '../workspace/VirtualizedCodeLocationRow'; -import {WorkspaceLocationNodeFragment} from '../workspace/types/WorkspaceContext.types'; +import {WorkspaceLocationNodeFragment} from '../workspace/types/WorkspaceQueries.types'; const flatten = (locationEntries: WorkspaceLocationNodeFragment[]) => { // Consider each loaded repository to be a "code location". diff --git a/js_modules/dagster-ui/packages/ui-core/src/overview/OverviewSensors.tsx b/js_modules/dagster-ui/packages/ui-core/src/overview/OverviewSensors.tsx index fea23ea18a732..922220e762798 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/overview/OverviewSensors.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/overview/OverviewSensors.tsx @@ -32,7 +32,6 @@ import {SENSOR_TYPE_META} from '../workspace/VirtualizedSensorRow'; import {WorkspaceContext} from '../workspace/WorkspaceContext'; import {buildRepoAddress} from '../workspace/buildRepoAddress'; import {repoAddressAsHumanString} from '../workspace/repoAddressAsString'; -import {RootWorkspaceQuery} from '../workspace/types/WorkspaceContext.types'; function toSetFilterValue(type: SensorType) { const label = SENSOR_TYPE_META[type].name; @@ -55,12 +54,7 @@ const SENSOR_TYPE_TO_FILTER: Partial { - const { - allRepos, - visibleRepos, - loading: workspaceLoading, - data: cachedData, - } = useContext(WorkspaceContext); + const {allRepos, visibleRepos, loading: workspaceLoading} = useContext(WorkspaceContext); const repoCount = allRepos.length; const [searchValue, setSearchValue] = useQueryPersistedState({ @@ -346,7 +340,7 @@ export const OverviewSensors = () => { ); }; -const buildBuckets = (data?: null | OverviewSensorsQuery | RootWorkspaceQuery) => { +const buildBuckets = (data?: null | OverviewSensorsQuery) => { if (data?.workspaceOrError.__typename !== 'Workspace') { return []; } diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/CodeLocationRowSet.tsx b/js_modules/dagster-ui/packages/ui-core/src/workspace/CodeLocationRowSet.tsx index 84c4a300e8935..a03837c974bf4 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/CodeLocationRowSet.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/CodeLocationRowSet.tsx @@ -19,7 +19,7 @@ import {RepositoryLocationNonBlockingErrorDialog} from './RepositoryLocationErro import {WorkspaceRepositoryLocationNode} from './WorkspaceContext'; import {buildRepoAddress} from './buildRepoAddress'; import {repoAddressAsHumanString} from './repoAddressAsString'; -import {WorkspaceDisplayMetadataFragment} from './types/WorkspaceContext.types'; +import {WorkspaceDisplayMetadataFragment} from './types/WorkspaceQueries.types'; import {workspacePathFromAddress} from './workspacePath'; import {showSharedToaster} from '../app/DomUtils'; import {useCopyToClipboard} from '../app/browser'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/VirtualizedCodeLocationRow.tsx b/js_modules/dagster-ui/packages/ui-core/src/workspace/VirtualizedCodeLocationRow.tsx index 7b3b8cbc90ac6..109e95b561d7a 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/VirtualizedCodeLocationRow.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/VirtualizedCodeLocationRow.tsx @@ -12,7 +12,7 @@ import {repoAddressAsHumanString} from './repoAddressAsString'; import { WorkspaceLocationNodeFragment, WorkspaceRepositoryFragment, -} from './types/WorkspaceContext.types'; +} from './types/WorkspaceQueries.types'; import {workspacePathFromAddress} from './workspacePath'; import {TimeFromNow} from '../ui/TimeFromNow'; import {HeaderCell, HeaderRow, RowCell} from '../ui/VirtualizedTable'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx index dd44d84b78524..591f49760692c 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx @@ -12,7 +12,7 @@ import { WorkspaceRepositoryFragment, WorkspaceScheduleFragment, WorkspaceSensorFragment, -} from './types/WorkspaceContext.types'; +} from './types/WorkspaceQueries.types'; import { CodeLocationStatusQuery, CodeLocationStatusQueryVariables, diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceQueries.tsx b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceQueries.tsx index 05f5c8434b2b7..234b6cb3c4d1f 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceQueries.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceQueries.tsx @@ -7,86 +7,105 @@ export const LOCATION_WORKSPACE_QUERY = gql` query LocationWorkspaceQuery($name: String!) { workspaceLocationEntryOrError(name: $name) { __typename - ... on WorkspaceLocationEntry { + ...WorkspaceLocationNode + } + } + + fragment WorkspaceLocationNode on WorkspaceLocationEntry { + id + name + loadStatus + displayMetadata { + ...WorkspaceDisplayMetadata + } + updatedTimestamp + featureFlags { + name + enabled + } + locationOrLoadError { + ... on RepositoryLocation { id - name - loadStatus - displayMetadata { - key - value - } - updatedTimestamp - featureFlags { - name - enabled - } - locationOrLoadError { - ... on RepositoryLocation { - id - name - dagsterLibraryVersions { - name - version - } - repositories { - ...RepositoryInfoFragment - id - name - pipelines { - id - name - isJob - isAssetJob - pipelineSnapshotId - } - schedules { - id - name - cronSchedule - executionTimezone - mode - pipelineName - scheduleState { - id - selectorId - status - } - } - sensors { - id - name - jobOriginId - targets { - mode - pipelineName - } - sensorState { - id - selectorId - status - } - sensorType - } - partitionSets { - id - mode - pipelineName - } - assetGroups { - id - groupName - } - allTopLevelResourceDetails { - id - name - } - } - } - ...PythonErrorFragment - } + ...WorkspaceLocation } + ...PythonErrorFragment + } + } + + fragment WorkspaceDisplayMetadata on RepositoryMetadata { + key + value + } + + fragment WorkspaceLocation on RepositoryLocation { + id + isReloadSupported + serverId + name + dagsterLibraryVersions { + name + version + } + repositories { + id + ...WorkspaceRepository } } + + fragment WorkspaceRepository on Repository { + id + name + pipelines { + id + name + isJob + isAssetJob + pipelineSnapshotId + } + schedules { + id + ...WorkspaceSchedule + } + sensors { + id + ...WorkspaceSensor + } + partitionSets { + id + mode + pipelineName + } + assetGroups { + id + groupName + } + allTopLevelResourceDetails { + id + name + } + ...RepositoryInfoFragment + } + + fragment WorkspaceSchedule on Schedule { + id + cronSchedule + executionTimezone + mode + name + pipelineName + } + + fragment WorkspaceSensor on Sensor { + id + jobOriginId + name + targets { + mode + pipelineName + } + sensorType + } + ${PYTHON_ERROR_FRAGMENT} ${REPOSITORY_INFO_FRAGMENT} `; diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/__tests__/getFeatureFlagForCodeLocation.test.tsx b/js_modules/dagster-ui/packages/ui-core/src/workspace/__tests__/getFeatureFlagForCodeLocation.test.tsx index 6c38a4ce3b699..0bfb1480559df 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/__tests__/getFeatureFlagForCodeLocation.test.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/__tests__/getFeatureFlagForCodeLocation.test.tsx @@ -1,6 +1,6 @@ import {buildFeatureFlag, buildWorkspaceLocationEntry} from '../../graphql/types'; import {getFeatureFlagForCodeLocation} from '../WorkspaceContext'; -import {WorkspaceLocationNodeFragment} from '../types/WorkspaceContext.types'; +import {WorkspaceLocationNodeFragment} from '../types/WorkspaceQueries.types'; describe('getFeatureFlagForCodeLocation', () => { const locationEntries: WorkspaceLocationNodeFragment[] = [ diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/types/WorkspaceContext.types.ts b/js_modules/dagster-ui/packages/ui-core/src/workspace/types/WorkspaceContext.types.ts deleted file mode 100644 index 79e87d591d4c2..0000000000000 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/types/WorkspaceContext.types.ts +++ /dev/null @@ -1,396 +0,0 @@ -// Generated GraphQL types, do not edit manually. - -import * as Types from '../../graphql/types'; - -export type RootWorkspaceQueryVariables = Types.Exact<{[key: string]: never}>; - -export type RootWorkspaceQuery = { - __typename: 'Query'; - workspaceOrError: - | { - __typename: 'PythonError'; - message: string; - stack: Array; - errorChain: Array<{ - __typename: 'ErrorChainLink'; - isExplicitLink: boolean; - error: {__typename: 'PythonError'; message: string; stack: Array}; - }>; - } - | { - __typename: 'Workspace'; - id: string; - locationEntries: Array<{ - __typename: 'WorkspaceLocationEntry'; - id: string; - name: string; - loadStatus: Types.RepositoryLocationLoadStatus; - updatedTimestamp: number; - displayMetadata: Array<{__typename: 'RepositoryMetadata'; key: string; value: string}>; - featureFlags: Array<{__typename: 'FeatureFlag'; name: string; enabled: boolean}>; - locationOrLoadError: - | { - __typename: 'PythonError'; - message: string; - stack: Array; - errorChain: Array<{ - __typename: 'ErrorChainLink'; - isExplicitLink: boolean; - error: {__typename: 'PythonError'; message: string; stack: Array}; - }>; - } - | { - __typename: 'RepositoryLocation'; - id: string; - isReloadSupported: boolean; - serverId: string | null; - name: string; - dagsterLibraryVersions: Array<{ - __typename: 'DagsterLibraryVersion'; - name: string; - version: string; - }> | null; - repositories: Array<{ - __typename: 'Repository'; - id: string; - name: string; - pipelines: Array<{ - __typename: 'Pipeline'; - id: string; - name: string; - isJob: boolean; - isAssetJob: boolean; - pipelineSnapshotId: string; - }>; - schedules: Array<{ - __typename: 'Schedule'; - id: string; - cronSchedule: string; - executionTimezone: string | null; - mode: string; - name: string; - pipelineName: string; - scheduleState: { - __typename: 'InstigationState'; - id: string; - selectorId: string; - status: Types.InstigationStatus; - }; - }>; - sensors: Array<{ - __typename: 'Sensor'; - id: string; - jobOriginId: string; - name: string; - sensorType: Types.SensorType; - targets: Array<{ - __typename: 'Target'; - mode: string; - pipelineName: string; - }> | null; - sensorState: { - __typename: 'InstigationState'; - id: string; - selectorId: string; - status: Types.InstigationStatus; - hasStartPermission: boolean; - hasStopPermission: boolean; - typeSpecificData: - | {__typename: 'ScheduleData'} - | {__typename: 'SensorData'; lastCursor: string | null} - | null; - }; - }>; - partitionSets: Array<{ - __typename: 'PartitionSet'; - id: string; - mode: string; - pipelineName: string; - }>; - assetGroups: Array<{__typename: 'AssetGroup'; id: string; groupName: string}>; - allTopLevelResourceDetails: Array<{ - __typename: 'ResourceDetails'; - id: string; - name: string; - }>; - location: {__typename: 'RepositoryLocation'; id: string; name: string}; - displayMetadata: Array<{ - __typename: 'RepositoryMetadata'; - key: string; - value: string; - }>; - }>; - } - | null; - }>; - }; -}; - -export type WorkspaceLocationNodeFragment = { - __typename: 'WorkspaceLocationEntry'; - id: string; - name: string; - loadStatus: Types.RepositoryLocationLoadStatus; - updatedTimestamp: number; - displayMetadata: Array<{__typename: 'RepositoryMetadata'; key: string; value: string}>; - featureFlags: Array<{__typename: 'FeatureFlag'; name: string; enabled: boolean}>; - locationOrLoadError: - | { - __typename: 'PythonError'; - message: string; - stack: Array; - errorChain: Array<{ - __typename: 'ErrorChainLink'; - isExplicitLink: boolean; - error: {__typename: 'PythonError'; message: string; stack: Array}; - }>; - } - | { - __typename: 'RepositoryLocation'; - id: string; - isReloadSupported: boolean; - serverId: string | null; - name: string; - dagsterLibraryVersions: Array<{ - __typename: 'DagsterLibraryVersion'; - name: string; - version: string; - }> | null; - repositories: Array<{ - __typename: 'Repository'; - id: string; - name: string; - pipelines: Array<{ - __typename: 'Pipeline'; - id: string; - name: string; - isJob: boolean; - isAssetJob: boolean; - pipelineSnapshotId: string; - }>; - schedules: Array<{ - __typename: 'Schedule'; - id: string; - cronSchedule: string; - executionTimezone: string | null; - mode: string; - name: string; - pipelineName: string; - scheduleState: { - __typename: 'InstigationState'; - id: string; - selectorId: string; - status: Types.InstigationStatus; - }; - }>; - sensors: Array<{ - __typename: 'Sensor'; - id: string; - jobOriginId: string; - name: string; - sensorType: Types.SensorType; - targets: Array<{__typename: 'Target'; mode: string; pipelineName: string}> | null; - sensorState: { - __typename: 'InstigationState'; - id: string; - selectorId: string; - status: Types.InstigationStatus; - hasStartPermission: boolean; - hasStopPermission: boolean; - typeSpecificData: - | {__typename: 'ScheduleData'} - | {__typename: 'SensorData'; lastCursor: string | null} - | null; - }; - }>; - partitionSets: Array<{ - __typename: 'PartitionSet'; - id: string; - mode: string; - pipelineName: string; - }>; - assetGroups: Array<{__typename: 'AssetGroup'; id: string; groupName: string}>; - allTopLevelResourceDetails: Array<{ - __typename: 'ResourceDetails'; - id: string; - name: string; - }>; - location: {__typename: 'RepositoryLocation'; id: string; name: string}; - displayMetadata: Array<{__typename: 'RepositoryMetadata'; key: string; value: string}>; - }>; - } - | null; -}; - -export type WorkspaceDisplayMetadataFragment = { - __typename: 'RepositoryMetadata'; - key: string; - value: string; -}; - -export type WorkspaceLocationFragment = { - __typename: 'RepositoryLocation'; - id: string; - isReloadSupported: boolean; - serverId: string | null; - name: string; - dagsterLibraryVersions: Array<{ - __typename: 'DagsterLibraryVersion'; - name: string; - version: string; - }> | null; - repositories: Array<{ - __typename: 'Repository'; - id: string; - name: string; - pipelines: Array<{ - __typename: 'Pipeline'; - id: string; - name: string; - isJob: boolean; - isAssetJob: boolean; - pipelineSnapshotId: string; - }>; - schedules: Array<{ - __typename: 'Schedule'; - id: string; - cronSchedule: string; - executionTimezone: string | null; - mode: string; - name: string; - pipelineName: string; - scheduleState: { - __typename: 'InstigationState'; - id: string; - selectorId: string; - status: Types.InstigationStatus; - }; - }>; - sensors: Array<{ - __typename: 'Sensor'; - id: string; - jobOriginId: string; - name: string; - sensorType: Types.SensorType; - targets: Array<{__typename: 'Target'; mode: string; pipelineName: string}> | null; - sensorState: { - __typename: 'InstigationState'; - id: string; - selectorId: string; - status: Types.InstigationStatus; - hasStartPermission: boolean; - hasStopPermission: boolean; - typeSpecificData: - | {__typename: 'ScheduleData'} - | {__typename: 'SensorData'; lastCursor: string | null} - | null; - }; - }>; - partitionSets: Array<{ - __typename: 'PartitionSet'; - id: string; - mode: string; - pipelineName: string; - }>; - assetGroups: Array<{__typename: 'AssetGroup'; id: string; groupName: string}>; - allTopLevelResourceDetails: Array<{__typename: 'ResourceDetails'; id: string; name: string}>; - location: {__typename: 'RepositoryLocation'; id: string; name: string}; - displayMetadata: Array<{__typename: 'RepositoryMetadata'; key: string; value: string}>; - }>; -}; - -export type WorkspaceRepositoryFragment = { - __typename: 'Repository'; - id: string; - name: string; - pipelines: Array<{ - __typename: 'Pipeline'; - id: string; - name: string; - isJob: boolean; - isAssetJob: boolean; - pipelineSnapshotId: string; - }>; - schedules: Array<{ - __typename: 'Schedule'; - id: string; - cronSchedule: string; - executionTimezone: string | null; - mode: string; - name: string; - pipelineName: string; - scheduleState: { - __typename: 'InstigationState'; - id: string; - selectorId: string; - status: Types.InstigationStatus; - }; - }>; - sensors: Array<{ - __typename: 'Sensor'; - id: string; - jobOriginId: string; - name: string; - sensorType: Types.SensorType; - targets: Array<{__typename: 'Target'; mode: string; pipelineName: string}> | null; - sensorState: { - __typename: 'InstigationState'; - id: string; - selectorId: string; - status: Types.InstigationStatus; - hasStartPermission: boolean; - hasStopPermission: boolean; - typeSpecificData: - | {__typename: 'ScheduleData'} - | {__typename: 'SensorData'; lastCursor: string | null} - | null; - }; - }>; - partitionSets: Array<{ - __typename: 'PartitionSet'; - id: string; - mode: string; - pipelineName: string; - }>; - assetGroups: Array<{__typename: 'AssetGroup'; id: string; groupName: string}>; - allTopLevelResourceDetails: Array<{__typename: 'ResourceDetails'; id: string; name: string}>; - location: {__typename: 'RepositoryLocation'; id: string; name: string}; - displayMetadata: Array<{__typename: 'RepositoryMetadata'; key: string; value: string}>; -}; - -export type WorkspaceScheduleFragment = { - __typename: 'Schedule'; - id: string; - cronSchedule: string; - executionTimezone: string | null; - mode: string; - name: string; - pipelineName: string; - scheduleState: { - __typename: 'InstigationState'; - id: string; - selectorId: string; - status: Types.InstigationStatus; - }; -}; - -export type WorkspaceSensorFragment = { - __typename: 'Sensor'; - id: string; - jobOriginId: string; - name: string; - sensorType: Types.SensorType; - targets: Array<{__typename: 'Target'; mode: string; pipelineName: string}> | null; - sensorState: { - __typename: 'InstigationState'; - id: string; - selectorId: string; - status: Types.InstigationStatus; - hasStartPermission: boolean; - hasStopPermission: boolean; - typeSpecificData: - | {__typename: 'ScheduleData'} - | {__typename: 'SensorData'; lastCursor: string | null} - | null; - }; -}; diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/types/WorkspaceQueries.types.ts b/js_modules/dagster-ui/packages/ui-core/src/workspace/types/WorkspaceQueries.types.ts index 518bcf2d173e1..524af40a236e7 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/types/WorkspaceQueries.types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/types/WorkspaceQueries.types.ts @@ -32,6 +32,8 @@ export type LocationWorkspaceQuery = { | { __typename: 'RepositoryLocation'; id: string; + isReloadSupported: boolean; + serverId: string | null; name: string; dagsterLibraryVersions: Array<{ __typename: 'DagsterLibraryVersion'; @@ -53,31 +55,19 @@ export type LocationWorkspaceQuery = { schedules: Array<{ __typename: 'Schedule'; id: string; - name: string; cronSchedule: string; executionTimezone: string | null; mode: string; + name: string; pipelineName: string; - scheduleState: { - __typename: 'InstigationState'; - id: string; - selectorId: string; - status: Types.InstigationStatus; - }; }>; sensors: Array<{ __typename: 'Sensor'; id: string; - name: string; jobOriginId: string; + name: string; sensorType: Types.SensorType; targets: Array<{__typename: 'Target'; mode: string; pipelineName: string}> | null; - sensorState: { - __typename: 'InstigationState'; - id: string; - selectorId: string; - status: Types.InstigationStatus; - }; }>; partitionSets: Array<{ __typename: 'PartitionSet'; @@ -91,6 +81,12 @@ export type LocationWorkspaceQuery = { id: string; name: string; }>; + location: {__typename: 'RepositoryLocation'; id: string; name: string}; + displayMetadata: Array<{ + __typename: 'RepositoryMetadata'; + key: string; + value: string; + }>; }>; } | null; @@ -98,6 +94,203 @@ export type LocationWorkspaceQuery = { | null; }; +export type WorkspaceLocationNodeFragment = { + __typename: 'WorkspaceLocationEntry'; + id: string; + name: string; + loadStatus: Types.RepositoryLocationLoadStatus; + updatedTimestamp: number; + displayMetadata: Array<{__typename: 'RepositoryMetadata'; key: string; value: string}>; + featureFlags: Array<{__typename: 'FeatureFlag'; name: string; enabled: boolean}>; + locationOrLoadError: + | { + __typename: 'PythonError'; + message: string; + stack: Array; + errorChain: Array<{ + __typename: 'ErrorChainLink'; + isExplicitLink: boolean; + error: {__typename: 'PythonError'; message: string; stack: Array}; + }>; + } + | { + __typename: 'RepositoryLocation'; + id: string; + isReloadSupported: boolean; + serverId: string | null; + name: string; + dagsterLibraryVersions: Array<{ + __typename: 'DagsterLibraryVersion'; + name: string; + version: string; + }> | null; + repositories: Array<{ + __typename: 'Repository'; + id: string; + name: string; + pipelines: Array<{ + __typename: 'Pipeline'; + id: string; + name: string; + isJob: boolean; + isAssetJob: boolean; + pipelineSnapshotId: string; + }>; + schedules: Array<{ + __typename: 'Schedule'; + id: string; + cronSchedule: string; + executionTimezone: string | null; + mode: string; + name: string; + pipelineName: string; + }>; + sensors: Array<{ + __typename: 'Sensor'; + id: string; + jobOriginId: string; + name: string; + sensorType: Types.SensorType; + targets: Array<{__typename: 'Target'; mode: string; pipelineName: string}> | null; + }>; + partitionSets: Array<{ + __typename: 'PartitionSet'; + id: string; + mode: string; + pipelineName: string; + }>; + assetGroups: Array<{__typename: 'AssetGroup'; id: string; groupName: string}>; + allTopLevelResourceDetails: Array<{ + __typename: 'ResourceDetails'; + id: string; + name: string; + }>; + location: {__typename: 'RepositoryLocation'; id: string; name: string}; + displayMetadata: Array<{__typename: 'RepositoryMetadata'; key: string; value: string}>; + }>; + } + | null; +}; + +export type WorkspaceDisplayMetadataFragment = { + __typename: 'RepositoryMetadata'; + key: string; + value: string; +}; + +export type WorkspaceLocationFragment = { + __typename: 'RepositoryLocation'; + id: string; + isReloadSupported: boolean; + serverId: string | null; + name: string; + dagsterLibraryVersions: Array<{ + __typename: 'DagsterLibraryVersion'; + name: string; + version: string; + }> | null; + repositories: Array<{ + __typename: 'Repository'; + id: string; + name: string; + pipelines: Array<{ + __typename: 'Pipeline'; + id: string; + name: string; + isJob: boolean; + isAssetJob: boolean; + pipelineSnapshotId: string; + }>; + schedules: Array<{ + __typename: 'Schedule'; + id: string; + cronSchedule: string; + executionTimezone: string | null; + mode: string; + name: string; + pipelineName: string; + }>; + sensors: Array<{ + __typename: 'Sensor'; + id: string; + jobOriginId: string; + name: string; + sensorType: Types.SensorType; + targets: Array<{__typename: 'Target'; mode: string; pipelineName: string}> | null; + }>; + partitionSets: Array<{ + __typename: 'PartitionSet'; + id: string; + mode: string; + pipelineName: string; + }>; + assetGroups: Array<{__typename: 'AssetGroup'; id: string; groupName: string}>; + allTopLevelResourceDetails: Array<{__typename: 'ResourceDetails'; id: string; name: string}>; + location: {__typename: 'RepositoryLocation'; id: string; name: string}; + displayMetadata: Array<{__typename: 'RepositoryMetadata'; key: string; value: string}>; + }>; +}; + +export type WorkspaceRepositoryFragment = { + __typename: 'Repository'; + id: string; + name: string; + pipelines: Array<{ + __typename: 'Pipeline'; + id: string; + name: string; + isJob: boolean; + isAssetJob: boolean; + pipelineSnapshotId: string; + }>; + schedules: Array<{ + __typename: 'Schedule'; + id: string; + cronSchedule: string; + executionTimezone: string | null; + mode: string; + name: string; + pipelineName: string; + }>; + sensors: Array<{ + __typename: 'Sensor'; + id: string; + jobOriginId: string; + name: string; + sensorType: Types.SensorType; + targets: Array<{__typename: 'Target'; mode: string; pipelineName: string}> | null; + }>; + partitionSets: Array<{ + __typename: 'PartitionSet'; + id: string; + mode: string; + pipelineName: string; + }>; + assetGroups: Array<{__typename: 'AssetGroup'; id: string; groupName: string}>; + allTopLevelResourceDetails: Array<{__typename: 'ResourceDetails'; id: string; name: string}>; + location: {__typename: 'RepositoryLocation'; id: string; name: string}; + displayMetadata: Array<{__typename: 'RepositoryMetadata'; key: string; value: string}>; +}; + +export type WorkspaceScheduleFragment = { + __typename: 'Schedule'; + id: string; + cronSchedule: string; + executionTimezone: string | null; + mode: string; + name: string; + pipelineName: string; +}; + +export type WorkspaceSensorFragment = { + __typename: 'Sensor'; + id: string; + jobOriginId: string; + name: string; + sensorType: Types.SensorType; + targets: Array<{__typename: 'Target'; mode: string; pipelineName: string}> | null; +}; + export type CodeLocationStatusQueryVariables = Types.Exact<{[key: string]: never}>; export type CodeLocationStatusQuery = { From e087d41bfa3162959bc012f3b2cc678996fa503d Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Wed, 5 Jun 2024 06:08:00 -0400 Subject: [PATCH 03/16] ts --- .../src/workspace/WorkspaceQueries.tsx | 16 +++- .../workspace/types/WorkspaceQueries.types.ts | 90 +++++++++++++++++++ 2 files changed, 104 insertions(+), 2 deletions(-) diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceQueries.tsx b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceQueries.tsx index 234b6cb3c4d1f..03469156203cb 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceQueries.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceQueries.tsx @@ -2,11 +2,11 @@ import {gql} from '@apollo/client'; import {REPOSITORY_INFO_FRAGMENT} from './RepositoryInformation'; import {PYTHON_ERROR_FRAGMENT} from '../app/PythonErrorFragment'; +import {SENSOR_SWITCH_FRAGMENT} from '../sensors/SensorSwitch'; export const LOCATION_WORKSPACE_QUERY = gql` query LocationWorkspaceQuery($name: String!) { workspaceLocationEntryOrError(name: $name) { - __typename ...WorkspaceLocationNode } } @@ -93,6 +93,11 @@ export const LOCATION_WORKSPACE_QUERY = gql` mode name pipelineName + scheduleState { + id + selectorId + status + } } fragment WorkspaceSensor on Sensor { @@ -103,9 +108,16 @@ export const LOCATION_WORKSPACE_QUERY = gql` mode pipelineName } + sensorState { + id + selectorId + status + ...BasicInstigationStateFragment + } sensorType + ...SensorSwitchFragment } - + ${SENSOR_SWITCH_FRAGMENT} ${PYTHON_ERROR_FRAGMENT} ${REPOSITORY_INFO_FRAGMENT} `; diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/types/WorkspaceQueries.types.ts b/js_modules/dagster-ui/packages/ui-core/src/workspace/types/WorkspaceQueries.types.ts index 524af40a236e7..720879ed0d097 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/types/WorkspaceQueries.types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/types/WorkspaceQueries.types.ts @@ -60,6 +60,12 @@ export type LocationWorkspaceQuery = { mode: string; name: string; pipelineName: string; + scheduleState: { + __typename: 'InstigationState'; + id: string; + selectorId: string; + status: Types.InstigationStatus; + }; }>; sensors: Array<{ __typename: 'Sensor'; @@ -68,6 +74,18 @@ export type LocationWorkspaceQuery = { name: string; sensorType: Types.SensorType; targets: Array<{__typename: 'Target'; mode: string; pipelineName: string}> | null; + sensorState: { + __typename: 'InstigationState'; + id: string; + selectorId: string; + status: Types.InstigationStatus; + hasStartPermission: boolean; + hasStopPermission: boolean; + typeSpecificData: + | {__typename: 'ScheduleData'} + | {__typename: 'SensorData'; lastCursor: string | null} + | null; + }; }>; partitionSets: Array<{ __typename: 'PartitionSet'; @@ -144,6 +162,12 @@ export type WorkspaceLocationNodeFragment = { mode: string; name: string; pipelineName: string; + scheduleState: { + __typename: 'InstigationState'; + id: string; + selectorId: string; + status: Types.InstigationStatus; + }; }>; sensors: Array<{ __typename: 'Sensor'; @@ -152,6 +176,18 @@ export type WorkspaceLocationNodeFragment = { name: string; sensorType: Types.SensorType; targets: Array<{__typename: 'Target'; mode: string; pipelineName: string}> | null; + sensorState: { + __typename: 'InstigationState'; + id: string; + selectorId: string; + status: Types.InstigationStatus; + hasStartPermission: boolean; + hasStopPermission: boolean; + typeSpecificData: + | {__typename: 'ScheduleData'} + | {__typename: 'SensorData'; lastCursor: string | null} + | null; + }; }>; partitionSets: Array<{ __typename: 'PartitionSet'; @@ -209,6 +245,12 @@ export type WorkspaceLocationFragment = { mode: string; name: string; pipelineName: string; + scheduleState: { + __typename: 'InstigationState'; + id: string; + selectorId: string; + status: Types.InstigationStatus; + }; }>; sensors: Array<{ __typename: 'Sensor'; @@ -217,6 +259,18 @@ export type WorkspaceLocationFragment = { name: string; sensorType: Types.SensorType; targets: Array<{__typename: 'Target'; mode: string; pipelineName: string}> | null; + sensorState: { + __typename: 'InstigationState'; + id: string; + selectorId: string; + status: Types.InstigationStatus; + hasStartPermission: boolean; + hasStopPermission: boolean; + typeSpecificData: + | {__typename: 'ScheduleData'} + | {__typename: 'SensorData'; lastCursor: string | null} + | null; + }; }>; partitionSets: Array<{ __typename: 'PartitionSet'; @@ -251,6 +305,12 @@ export type WorkspaceRepositoryFragment = { mode: string; name: string; pipelineName: string; + scheduleState: { + __typename: 'InstigationState'; + id: string; + selectorId: string; + status: Types.InstigationStatus; + }; }>; sensors: Array<{ __typename: 'Sensor'; @@ -259,6 +319,18 @@ export type WorkspaceRepositoryFragment = { name: string; sensorType: Types.SensorType; targets: Array<{__typename: 'Target'; mode: string; pipelineName: string}> | null; + sensorState: { + __typename: 'InstigationState'; + id: string; + selectorId: string; + status: Types.InstigationStatus; + hasStartPermission: boolean; + hasStopPermission: boolean; + typeSpecificData: + | {__typename: 'ScheduleData'} + | {__typename: 'SensorData'; lastCursor: string | null} + | null; + }; }>; partitionSets: Array<{ __typename: 'PartitionSet'; @@ -280,6 +352,12 @@ export type WorkspaceScheduleFragment = { mode: string; name: string; pipelineName: string; + scheduleState: { + __typename: 'InstigationState'; + id: string; + selectorId: string; + status: Types.InstigationStatus; + }; }; export type WorkspaceSensorFragment = { @@ -289,6 +367,18 @@ export type WorkspaceSensorFragment = { name: string; sensorType: Types.SensorType; targets: Array<{__typename: 'Target'; mode: string; pipelineName: string}> | null; + sensorState: { + __typename: 'InstigationState'; + id: string; + selectorId: string; + status: Types.InstigationStatus; + hasStartPermission: boolean; + hasStopPermission: boolean; + typeSpecificData: + | {__typename: 'ScheduleData'} + | {__typename: 'SensorData'; lastCursor: string | null} + | null; + }; }; export type CodeLocationStatusQueryVariables = Types.Exact<{[key: string]: never}>; From 6b8575252e3765afd376b5f591f341744e35bb6b Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Wed, 5 Jun 2024 13:54:14 -0400 Subject: [PATCH 04/16] ts --- .../ui-core/src/workspace/WorkspaceContext.tsx | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx index 591f49760692c..c2be3c8139b30 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx @@ -7,18 +7,16 @@ import {buildRepoAddress} from './buildRepoAddress'; import {findRepoContainingPipeline} from './findRepoContainingPipeline'; import {RepoAddress} from './types'; import { + CodeLocationStatusQuery, + CodeLocationStatusQueryVariables, + LocationWorkspaceQuery, + LocationWorkspaceQueryVariables, WorkspaceLocationFragment, WorkspaceLocationNodeFragment, WorkspaceRepositoryFragment, WorkspaceScheduleFragment, WorkspaceSensorFragment, } from './types/WorkspaceQueries.types'; -import { - CodeLocationStatusQuery, - CodeLocationStatusQueryVariables, - LocationWorkspaceQuery, - LocationWorkspaceQueryVariables, -} from './types/WorkspaceQueries.types'; import {AppContext} from '../app/AppContext'; import {useRefreshAtInterval} from '../app/QueryRefresh'; import {PythonErrorFragment} from '../app/types/PythonErrorFragment.types'; From 80d60d24b04d13d24674934c65cda30f3b2c4658 Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Wed, 5 Jun 2024 14:20:39 -0400 Subject: [PATCH 05/16] update test --- ...LaunchAssetChoosePartitionsDialog.test.tsx | 27 ++++++++++--------- .../nav/__tests__/useDaemonStatus.test.tsx | 2 -- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/__tests__/LaunchAssetChoosePartitionsDialog.test.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/__tests__/LaunchAssetChoosePartitionsDialog.test.tsx index 6870d3938ca42..b001d6fb59497 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/__tests__/LaunchAssetChoosePartitionsDialog.test.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/__tests__/LaunchAssetChoosePartitionsDialog.test.tsx @@ -19,6 +19,7 @@ import { AddDynamicPartitionMutationVariables, } from '../../partitions/types/CreatePartitionDialog.types'; import {buildMutationMock, buildQueryMock, getMockResultFn} from '../../testing/mocking'; +import {WorkspaceProvider} from '../../workspace/WorkspaceContext'; import {buildWorkspaceMocks} from '../../workspace/__fixtures__/Workspace.fixtures'; import {buildRepoAddress} from '../../workspace/buildRepoAddress'; import {LaunchAssetChoosePartitionsDialog} from '../LaunchAssetChoosePartitionsDialog'; @@ -94,18 +95,20 @@ describe('launchAssetChoosePartitionsDialog', () => { ...workspaceMocks, ]} > - {}} - repoAddress={buildRepoAddress('test', 'test')} - target={{ - jobName: '__ASSET_JOB_0', - partitionSetName: '__ASSET_JOB_0_partition_set', - type: 'job', - }} - assets={[assetA, assetB]} - upstreamAssetKeys={[]} - /> + + {}} + repoAddress={buildRepoAddress('test', 'test')} + target={{ + jobName: '__ASSET_JOB_0', + partitionSetName: '__ASSET_JOB_0_partition_set', + type: 'job', + }} + assets={[assetA, assetB]} + upstreamAssetKeys={[]} + /> + , ); diff --git a/js_modules/dagster-ui/packages/ui-core/src/nav/__tests__/useDaemonStatus.test.tsx b/js_modules/dagster-ui/packages/ui-core/src/nav/__tests__/useDaemonStatus.test.tsx index f706d2a38b9e8..f143ace001c66 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/nav/__tests__/useDaemonStatus.test.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/nav/__tests__/useDaemonStatus.test.tsx @@ -58,7 +58,6 @@ describe('useDaemonStatus', () => { }); it('does surface scheduler errors if there is a running schedule', async () => { - (window as any).__debug = true; const daemonHealth = [{daemonType: 'SCHEDULER', healthy: false, required: true}]; const {result} = renderHook(() => useDaemonStatus(), { @@ -82,7 +81,6 @@ describe('useDaemonStatus', () => { render(
{result.current?.content}
); expect(screen.getByText(/1 daemon not running/i)).toBeVisible(); }); - (window as any).__debug = false; }); }); From d2ee1cb00fa8e7d0c3b631fe848998f564adc3a5 Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Wed, 5 Jun 2024 19:04:25 -0400 Subject: [PATCH 06/16] delete indexeddb for removed locations --- .../packages/ui-core/src/workspace/WorkspaceContext.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx index c2be3c8139b30..50493c124fa71 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx @@ -186,11 +186,12 @@ export const WorkspaceProvider = ({children}: {children: React.ReactNode}) => { const copy = {...locationsData}; locationsRemoved.forEach((loc) => { delete copy[loc.name]; + indexedDB.deleteDatabase(`${localCacheIdPrefix}/${locationWorkspaceKey(loc.name)}`); }); return copy; }); } - }, [locationsRemoved]); + }, [localCacheIdPrefix, locationsRemoved]); const locationEntries = useMemo( () => From 89b9f07a8ba52f8de3b91393abfd3426b00c9ca5 Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Thu, 6 Jun 2024 03:07:48 -0400 Subject: [PATCH 07/16] missing fragment --- .../packages/ui-core/src/workspace/WorkspaceQueries.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceQueries.tsx b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceQueries.tsx index 03469156203cb..cc253de00c2b2 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceQueries.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceQueries.tsx @@ -2,6 +2,7 @@ import {gql} from '@apollo/client'; import {REPOSITORY_INFO_FRAGMENT} from './RepositoryInformation'; import {PYTHON_ERROR_FRAGMENT} from '../app/PythonErrorFragment'; +import {BASIC_INSTIGATION_STATE_FRAGMENT} from '../overview/BasicInstigationStateFragment'; import {SENSOR_SWITCH_FRAGMENT} from '../sensors/SensorSwitch'; export const LOCATION_WORKSPACE_QUERY = gql` @@ -117,6 +118,7 @@ export const LOCATION_WORKSPACE_QUERY = gql` sensorType ...SensorSwitchFragment } + ${BASIC_INSTIGATION_STATE_FRAGMENT} ${SENSOR_SWITCH_FRAGMENT} ${PYTHON_ERROR_FRAGMENT} ${REPOSITORY_INFO_FRAGMENT} From 79eabd83e7364487a53cd855c5f41afed6548e40 Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Thu, 6 Jun 2024 04:31:41 -0400 Subject: [PATCH 08/16] comment --- .../ui-core/src/runs/HourlyDataCache/HourlyDataCache.tsx | 2 +- .../ui-core/src/search/useIndexedDBCachedQuery.tsx | 2 +- .../packages/ui-core/src/workspace/WorkspaceContext.tsx | 9 +++++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/HourlyDataCache/HourlyDataCache.tsx b/js_modules/dagster-ui/packages/ui-core/src/runs/HourlyDataCache/HourlyDataCache.tsx index eea19f4de4c50..b9402e606a7e8 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/HourlyDataCache/HourlyDataCache.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/HourlyDataCache/HourlyDataCache.tsx @@ -7,7 +7,7 @@ export const ONE_HOUR_S = 60 * 60; type Subscription = (data: T[]) => void; export const defaultOptions = { - expiry: new Date('3000-01-01'), // never expire, + expiry: new Date('3030-01-01'), // never expire, }; export class HourlyDataCache { diff --git a/js_modules/dagster-ui/packages/ui-core/src/search/useIndexedDBCachedQuery.tsx b/js_modules/dagster-ui/packages/ui-core/src/search/useIndexedDBCachedQuery.tsx index 914e688edd950..c0bdbb748034c 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/search/useIndexedDBCachedQuery.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/search/useIndexedDBCachedQuery.tsx @@ -35,7 +35,7 @@ export class CacheManager { } set(data: TQuery, version: number): Promise { - return this.cache.set('cache', {data, version}, {expiry: new Date('3000-01-01')}); + return this.cache.set('cache', {data, version}, {expiry: new Date('3030-01-01')}); } } diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx index 50493c124fa71..94413fe002f34 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx @@ -99,6 +99,15 @@ export const WorkspaceProvider = ({children}: {children: React.ReactNode}) => { } didInitiateFetchFromCache.current = true; (async () => { + /** + * 1. Load the cached code location status query + * 2. Load the cached data for those locations + * 3. Set the cached data to `locationsData` state + * 4. Set prevLocations equal to these cached locations so that we can check if they + * have changed after the next call to codeLocationStatusQuery + * 5. set didLoadCachedData to true to unblock the `locationsToFetch` memo so that it can compare + * the latest codeLocationStatusQuery result to what was in the cache. + */ const data = await getCachedData({ key: `${localCacheIdPrefix}/${CODE_LOCATION_STATUS_QUERY_KEY}`, version: CODE_LOCATION_STATUS_QUERY_VERSION, From 08a85d1c3daf4048e815fdbea28df8914297ae6b Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Thu, 6 Jun 2024 23:21:06 -0400 Subject: [PATCH 09/16] tests --- .../backfill/__tests__/BackfillTable.test.tsx | 11 +- .../LeftNavRepositorySection.test.tsx | 19 +- .../src/search/useIndexedDBCachedQuery.tsx | 134 +++--- .../src/workspace/WorkspaceContext.tsx | 112 +++-- .../__fixtures__/Workspace.fixtures.ts | 26 +- .../__tests__/WorkspaceContext.test.tsx | 433 ++++++++++++++++++ 6 files changed, 626 insertions(+), 109 deletions(-) create mode 100644 js_modules/dagster-ui/packages/ui-core/src/workspace/__tests__/WorkspaceContext.test.tsx diff --git a/js_modules/dagster-ui/packages/ui-core/src/instance/backfill/__tests__/BackfillTable.test.tsx b/js_modules/dagster-ui/packages/ui-core/src/instance/backfill/__tests__/BackfillTable.test.tsx index fce0aa4dc0331..d0c0e314a095b 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/instance/backfill/__tests__/BackfillTable.test.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/instance/backfill/__tests__/BackfillTable.test.tsx @@ -1,6 +1,5 @@ import {MockedProvider} from '@apollo/client/testing'; -import {render, screen} from '@testing-library/react'; -import userEvent from '@testing-library/user-event'; +import {act, render, screen, waitFor} from '@testing-library/react'; import {MemoryRouter} from 'react-router-dom'; import * as Alerting from '../../../app/CustomAlertProvider'; @@ -28,7 +27,11 @@ describe('BackfillTable', () => { expect(screen.getByRole('table')).toBeVisible(); const statusLabel = await screen.findByText('Failed'); - await userEvent.click(statusLabel); - expect(Alerting.showCustomAlert).toHaveBeenCalled(); + act(() => { + statusLabel.click(); + }); + await waitFor(() => { + expect(Alerting.showCustomAlert).toHaveBeenCalled(); + }); }); }); diff --git a/js_modules/dagster-ui/packages/ui-core/src/nav/__tests__/LeftNavRepositorySection.test.tsx b/js_modules/dagster-ui/packages/ui-core/src/nav/__tests__/LeftNavRepositorySection.test.tsx index cc1f3f43f7dda..83b411950e634 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/nav/__tests__/LeftNavRepositorySection.test.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/nav/__tests__/LeftNavRepositorySection.test.tsx @@ -29,21 +29,20 @@ describe('Repository options', () => { let nativeGBRC: any; - beforeAll(() => { + beforeEach(() => { + window.localStorage.clear(); nativeGBRC = window.Element.prototype.getBoundingClientRect; window.Element.prototype.getBoundingClientRect = jest .fn() .mockReturnValue({height: 400, width: 400}); }); - afterAll(() => { - window.Element.prototype.getBoundingClientRect = nativeGBRC; - }); - afterEach(() => { + window.Element.prototype.getBoundingClientRect = nativeGBRC; window.localStorage.clear(); __resetForJest(); jest.resetModules(); + jest.resetAllMocks(); }); it('Correctly displays the current repository state', async () => { @@ -128,18 +127,14 @@ describe('Repository options', () => { const loremHeader = await screen.findByRole('button', {name: /lorem/i}); expect(loremHeader).toBeVisible(); - const fooHeader = screen.getByRole('button', {name: /foo/i}); + const fooHeader = await waitFor(() => screen.getByRole('button', {name: /foo/i})); expect(fooHeader).toBeVisible(); - const dunderHeader = screen.getByRole('button', {name: /abc_location/i}); + const dunderHeader = await waitFor(() => screen.getByRole('button', {name: /abc_location/i})); expect(dunderHeader).toBeVisible(); await userEvent.click(loremHeader); - await userEvent.click(fooHeader); - await userEvent.click(dunderHeader); - await waitFor(() => { - // Twelve jobs total. No repo name link since multiple repos are visible. - expect(screen.queryAllByRole('link')).toHaveLength(12); + expect(screen.queryAllByRole('link')).toHaveLength(6); }); }); diff --git a/js_modules/dagster-ui/packages/ui-core/src/search/useIndexedDBCachedQuery.tsx b/js_modules/dagster-ui/packages/ui-core/src/search/useIndexedDBCachedQuery.tsx index c0bdbb748034c..d68344bf138c5 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/search/useIndexedDBCachedQuery.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/search/useIndexedDBCachedQuery.tsx @@ -1,26 +1,21 @@ import {ApolloClient, DocumentNode, OperationVariables, useApolloClient} from '@apollo/client'; import {cache} from 'idb-lru-cache'; import memoize from 'lodash/memoize'; -import React, {useCallback} from 'react'; +import React, {createContext, useCallback, useContext} from 'react'; type CacheData = { data: TQuery; version: number; }; -let fetchState: Record< - string, - { - onFetched: ((value: any) => void)[]; - } -> = {}; +export const KEY_PREFIX = 'indexdbQueryCache:'; export class CacheManager { private cache: ReturnType>>; private key: string; constructor(key: string) { - this.key = `indexdbQueryCache:${key}`; + this.key = `${KEY_PREFIX}${key}`; this.cache = cache>({dbName: this.key, maxCount: 1}); } @@ -39,10 +34,6 @@ export class CacheManager { } } -const getCacheManager = memoize((key: string) => { - return new CacheManager(key); -}); - interface QueryHookParams { key: string; query: DocumentNode; @@ -61,6 +52,8 @@ export function useIndexedDBCachedQuery(null); const [loading, setLoading] = React.useState(true); + const getData = useGetData(); + const fetch = useCallback( async (bypassCache = false) => { setLoading(true); @@ -75,7 +68,7 @@ export function useIndexedDBCachedQuery { @@ -98,56 +91,91 @@ interface FetchParams { bypassCache?: boolean; } -export async function getData({ - client, - key, - query, - variables, - version, - bypassCache = false, -}: FetchParams): Promise { - const cacheManager = getCacheManager(key); - - if (!bypassCache) { - const cachedData = await cacheManager.get(version); - if (cachedData !== null) { - return cachedData; - } - } +export function useGetData() { + const {getCacheManager, fetchState} = useContext(IndexedDBCacheContext); + + return useCallback( + async ({ + client, + key, + query, + variables, + version, + bypassCache = false, + }: FetchParams): Promise => { + const cacheManager = getCacheManager(key); + + if (!bypassCache) { + const cachedData = await cacheManager.get(version); + if (cachedData !== null) { + return cachedData; + } + } - const currentState = fetchState[key]; - // Handle concurrent fetch requests - if (currentState) { - return new Promise((resolve) => { - currentState!.onFetched.push(resolve as any); - }); - } + const currentState = fetchState[key]; + // Handle concurrent fetch requests + if (currentState) { + return new Promise((resolve) => { + currentState!.onFetched.push(resolve as any); + }); + } - const state = {onFetched: [] as ((value: any) => void)[]}; - fetchState[key] = state; + const state = {onFetched: [] as ((value: any) => void)[]}; + fetchState[key] = state; - const queryResult = await client.query({ - query, - variables, - fetchPolicy: 'no-cache', - }); + const queryResult = await client.query({ + query, + variables, + fetchPolicy: 'no-cache', + }); - const {data} = queryResult; - await cacheManager.set(data, version); + const {data} = queryResult; + await cacheManager.set(data, version); - const onFetchedHandlers = state.onFetched; - delete fetchState[key]; // Clean up fetch state after handling + const onFetchedHandlers = state.onFetched; + if (fetchState[key] === state) { + delete fetchState[key]; // Clean up fetch state after handling + } - onFetchedHandlers.forEach((handler) => handler(data)); // Notify all waiting fetches + onFetchedHandlers.forEach((handler) => handler(data)); // Notify all waiting fetches - return data; + return data; + }, + // eslint-disable-next-line react-hooks/exhaustive-deps + [], + ); } -export async function getCachedData({key, version}: {key: string; version: number}) { - const cacheManager = getCacheManager(key); - return await cacheManager.get(version); +export function useGetCachedData() { + const {getCacheManager} = useContext(IndexedDBCacheContext); + + return useCallback( + async ({key, version}: {key: string; version: number}) => { + const cacheManager = getCacheManager(key); + return await cacheManager.get(version); + }, + // eslint-disable-next-line react-hooks/exhaustive-deps + [], + ); +} + +const contextValue = createIndexedDBCacheContextValue(); +export const IndexedDBCacheContext = createContext(contextValue); + +export function createIndexedDBCacheContextValue() { + return { + getCacheManager: memoize((key: string) => { + return new CacheManager(key); + }), + fetchState: {} as Record< + string, + { + onFetched: ((value: any) => void)[]; + } + >, + }; } export const __resetForJest = () => { - fetchState = {}; + Object.assign(contextValue, createIndexedDBCacheContextValue()); }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx index 94413fe002f34..cec5528f93134 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx @@ -1,6 +1,7 @@ import {useApolloClient} from '@apollo/client'; import sortBy from 'lodash/sortBy'; -import React, {useCallback, useContext, useEffect, useMemo, useRef, useState} from 'react'; +import React, {useCallback, useContext, useLayoutEffect, useMemo, useRef, useState} from 'react'; +import {useSetRecoilState} from 'recoil'; import {CODE_LOCATION_STATUS_QUERY, LOCATION_WORKSPACE_QUERY} from './WorkspaceQueries'; import {buildRepoAddress} from './buildRepoAddress'; @@ -23,11 +24,16 @@ import {PythonErrorFragment} from '../app/types/PythonErrorFragment.types'; import {PipelineSelector} from '../graphql/types'; import {useStateWithStorage} from '../hooks/useStateWithStorage'; import {useUpdatingRef} from '../hooks/useUpdatingRef'; -import {getCachedData, getData, useIndexedDBCachedQuery} from '../search/useIndexedDBCachedQuery'; - -const CODE_LOCATION_STATUS_QUERY_KEY = 'CodeLocationStatusQuery'; -const CODE_LOCATION_STATUS_QUERY_VERSION = 3; -const LOCATION_WORKSPACE_QUERY_VERSION = 5; +import {codeLocationStatusAtom} from '../nav/useCodeLocationsStatus'; +import { + useGetCachedData, + useGetData, + useIndexedDBCachedQuery, +} from '../search/useIndexedDBCachedQuery'; + +export const CODE_LOCATION_STATUS_QUERY_KEY = '/CodeLocationStatusQuery'; +export const CODE_LOCATION_STATUS_QUERY_VERSION = 1; +export const LOCATION_WORKSPACE_QUERY_VERSION = 1; type Repository = WorkspaceRepositoryFragment; type RepositoryLocation = WorkspaceLocationFragment; @@ -69,8 +75,21 @@ export const WorkspaceProvider = ({children}: {children: React.ReactNode}) => { >({ query: CODE_LOCATION_STATUS_QUERY, version: CODE_LOCATION_STATUS_QUERY_VERSION, - key: `${localCacheIdPrefix}/${CODE_LOCATION_STATUS_QUERY_KEY}`, + key: `${localCacheIdPrefix}${CODE_LOCATION_STATUS_QUERY_KEY}`, }); + if (typeof jest === 'undefined') { + // Only do this outside of jest for now so that we don't need to add RecoilRoot around everything... + // we will switch to jotai at some point instead... which doesnt require a + // eslint-disable-next-line react-hooks/rules-of-hooks + const setCodeLocationStatusAtom = useSetRecoilState(codeLocationStatusAtom); + // eslint-disable-next-line react-hooks/rules-of-hooks + useLayoutEffect(() => { + if (codeLocationStatusQueryResult.data) { + setCodeLocationStatusAtom(codeLocationStatusQueryResult.data); + } + }, [codeLocationStatusQueryResult.data, setCodeLocationStatusAtom]); + } + const fetch = codeLocationStatusQueryResult.fetch; useRefreshAtInterval({ refresh: useCallback(async () => { @@ -80,7 +99,7 @@ export const WorkspaceProvider = ({children}: {children: React.ReactNode}) => { leading: true, }); - const {data} = codeLocationStatusQueryResult; + const {data, loading: loadingCodeLocationStatus} = codeLocationStatusQueryResult; const locations = useMemo(() => getLocations(data), [data]); const prevLocations = useRef({}); @@ -92,13 +111,16 @@ export const WorkspaceProvider = ({children}: {children: React.ReactNode}) => { Record >({}); - useEffect(() => { + const getCachedData = useGetCachedData(); + const getData = useGetData(); + + useLayoutEffect(() => { // Load data from the cache if (didInitiateFetchFromCache.current) { return; } didInitiateFetchFromCache.current = true; - (async () => { + new Promise(async (res) => { /** * 1. Load the cached code location status query * 2. Load the cached data for those locations @@ -109,7 +131,7 @@ export const WorkspaceProvider = ({children}: {children: React.ReactNode}) => { * the latest codeLocationStatusQuery result to what was in the cache. */ const data = await getCachedData({ - key: `${localCacheIdPrefix}/${CODE_LOCATION_STATUS_QUERY_KEY}`, + key: `${localCacheIdPrefix}${CODE_LOCATION_STATUS_QUERY_KEY}`, version: CODE_LOCATION_STATUS_QUERY_VERSION, }); const cachedLocations = getLocations(data); @@ -118,7 +140,7 @@ export const WorkspaceProvider = ({children}: {children: React.ReactNode}) => { await Promise.all([ ...Object.values(cachedLocations).map(async (location) => { const locationData = await getCachedData({ - key: `${localCacheIdPrefix}/${locationWorkspaceKey(location.name)}`, + key: `${localCacheIdPrefix}${locationWorkspaceKey(location.name)}`, version: LOCATION_WORKSPACE_QUERY_VERSION, }); const entry = locationData?.workspaceLocationEntryOrError; @@ -138,9 +160,11 @@ export const WorkspaceProvider = ({children}: {children: React.ReactNode}) => { }), ]); prevLocations.current = prevCachedLocations; + res(void 0); + }).then(() => { setDidLoadCachedData(true); - })(); - }, [localCacheIdPrefix, locations]); + }); + }, [getCachedData, localCacheIdPrefix, locations]); const client = useApolloClient(); @@ -149,7 +173,7 @@ export const WorkspaceProvider = ({children}: {children: React.ReactNode}) => { const locationData = await getData({ client, query: LOCATION_WORKSPACE_QUERY, - key: `${localCacheIdPrefix}/${locationWorkspaceKey(name)}`, + key: `${localCacheIdPrefix}${locationWorkspaceKey(name)}`, version: LOCATION_WORKSPACE_QUERY_VERSION, variables: { name, @@ -179,23 +203,42 @@ export const WorkspaceProvider = ({children}: {children: React.ReactNode}) => { return toFetch; }, [locations, didLoadCachedData]); - useEffect(() => { - locationsToFetch.forEach(async (location) => { - refetchLocation(location.name); + const [isRefetching, setIsRefetching] = useState(false); + useLayoutEffect(() => { + if (!locationsToFetch.length) { + return; + } + setIsRefetching(true); + Promise.all( + locationsToFetch.map(async (location) => { + return await refetchLocation(location.name); + }), + ).then(() => { + setIsRefetching(false); }); }, [refetchLocation, locationsToFetch]); - const locationsRemoved = useMemo(() => { - return Object.values(prevLocations.current).filter((loc) => !locations[loc.name]); - }, [locations]); + const locationsRemoved = useMemo( + () => + Array.from( + new Set([ + ...Object.values(prevLocations.current).filter((loc) => !locations[loc.name]), + ...Object.values(locationsData).filter( + (loc): loc is WorkspaceLocationNodeFragment => + loc.__typename === 'WorkspaceLocationEntry' && !locations[loc.name], + ), + ]), + ), + [locations, locationsData], + ); - useEffect(() => { + useLayoutEffect(() => { if (locationsRemoved.length) { setLocationsData((locationsData) => { const copy = {...locationsData}; locationsRemoved.forEach((loc) => { delete copy[loc.name]; - indexedDB.deleteDatabase(`${localCacheIdPrefix}/${locationWorkspaceKey(loc.name)}`); + indexedDB.deleteDatabase(`${localCacheIdPrefix}${locationWorkspaceKey(loc.name)}`); }); return copy; }); @@ -211,7 +254,7 @@ export const WorkspaceProvider = ({children}: {children: React.ReactNode}) => { [locationsData], ); - const {allRepos} = React.useMemo(() => { + const allRepos = React.useMemo(() => { let allRepos: DagsterRepoOption[] = []; allRepos = sortBy( @@ -220,9 +263,7 @@ export const WorkspaceProvider = ({children}: {children: React.ReactNode}) => { return accum; } const repositoryLocation = locationEntry.locationOrLoadError; - const reposForLocation = repositoryLocation.repositories.map((repository) => { - return {repository, repositoryLocation}; - }); + const reposForLocation = repoLocationToRepos(repositoryLocation); return [...accum, ...reposForLocation]; }, [] as DagsterRepoOption[]), @@ -230,7 +271,7 @@ export const WorkspaceProvider = ({children}: {children: React.ReactNode}) => { (r) => `${r.repositoryLocation.name}:${r.repository.name}`, ); - return {allRepos}; + return allRepos; }, [locationEntries]); const {visibleRepos, toggleVisible, setVisible, setHidden} = useVisibleRepos(allRepos); @@ -246,7 +287,9 @@ export const WorkspaceProvider = ({children}: {children: React.ReactNode}) => { return ( { accum[loc.name] = loc; @@ -277,8 +321,8 @@ function getLocations(d: CodeLocationStatusQuery | undefined | null) { ); } -function locationWorkspaceKey(name: string) { - return `LocationWorkspace/${name}`; +export function locationWorkspaceKey(name: string) { + return `/LocationWorkspace/${name}`; } /** @@ -467,3 +511,9 @@ export const buildPipelineSelector = ( export const optionToRepoAddress = (option: DagsterRepoOption) => buildRepoAddress(option.repository.name, option.repository.location.name); + +export function repoLocationToRepos(repositoryLocation: RepositoryLocation) { + return repositoryLocation.repositories.map((repository) => { + return {repository, repositoryLocation}; + }); +} diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/__fixtures__/Workspace.fixtures.ts b/js_modules/dagster-ui/packages/ui-core/src/workspace/__fixtures__/Workspace.fixtures.ts index 34a1ac7bfed25..a90580d1959a7 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/__fixtures__/Workspace.fixtures.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/__fixtures__/Workspace.fixtures.ts @@ -17,6 +17,7 @@ import { export const buildCodeLocationsStatusQuery = ( entries: WorkspaceLocationStatusEntry[], + options: Partial> = {}, ): MockedResponse => { return buildQueryMock({ query: CODE_LOCATION_STATUS_QUERY, @@ -26,11 +27,25 @@ export const buildCodeLocationsStatusQuery = ( entries, }), }, + ...options, }); }; -export const buildWorkspaceMocks = (entries: WorkspaceLocationEntry[]) => { +export const buildWorkspaceMocks = ( + entries: WorkspaceLocationEntry[], + options: Partial> = {}, +) => { return [ + buildCodeLocationsStatusQuery( + entries.map((entry) => + buildWorkspaceLocationStatusEntry({ + ...entry, + updateTimestamp: entry.updatedTimestamp, + __typename: 'WorkspaceLocationStatusEntry', + }), + ), + options, + ), ...entries.map((entry) => buildQueryMock({ query: LOCATION_WORKSPACE_QUERY, @@ -40,15 +55,8 @@ export const buildWorkspaceMocks = (entries: WorkspaceLocationEntry[]) => { data: { workspaceLocationEntryOrError: entry, }, + ...options, }), ), - buildCodeLocationsStatusQuery( - entries.map((entry) => - buildWorkspaceLocationStatusEntry({ - ...entry, - __typename: 'WorkspaceLocationStatusEntry', - }), - ), - ), ]; }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/__tests__/WorkspaceContext.test.tsx b/js_modules/dagster-ui/packages/ui-core/src/workspace/__tests__/WorkspaceContext.test.tsx new file mode 100644 index 0000000000000..51198496288b8 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/__tests__/WorkspaceContext.test.tsx @@ -0,0 +1,433 @@ +jest.useFakeTimers(); + +import {MockedProvider, MockedResponse} from '@apollo/client/testing'; +import {act, renderHook, waitFor} from '@testing-library/react'; +import {cache} from 'idb-lru-cache'; +import {useContext} from 'react'; +import {RecoilRoot} from 'recoil'; + +import {AppContext} from '../../app/AppContext'; +import { + buildRepository, + buildRepositoryLocation, + buildWorkspaceLocationEntry, +} from '../../graphql/types'; +import { + IndexedDBCacheContext, + KEY_PREFIX, + createIndexedDBCacheContextValue, +} from '../../search/useIndexedDBCachedQuery'; +import {getMockResultFn} from '../../testing/mocking'; +import { + CODE_LOCATION_STATUS_QUERY_KEY, + CODE_LOCATION_STATUS_QUERY_VERSION, + LOCATION_WORKSPACE_QUERY_VERSION, + WorkspaceContext, + WorkspaceProvider, + locationWorkspaceKey, + repoLocationToRepos, +} from '../WorkspaceContext'; +import {buildWorkspaceMocks} from '../__fixtures__/Workspace.fixtures'; + +const mockCache = cache as any; + +const mockedCacheStore: Record = {}; + +jest.mock('idb-lru-cache', () => { + return { + cache: jest.fn(({dbName}) => { + if (!mockedCacheStore[dbName]) { + mockedCacheStore[dbName] = { + has: jest.fn(), + get: jest.fn(), + set: jest.fn(), + }; + } + return mockedCacheStore[dbName]; + }), + }; +}); + +afterEach(async () => { + jest.resetModules(); + jest.clearAllMocks(); + jest.clearAllTimers(); +}); +const LOCAL_CACHE_ID_PREFIX = 'test'; +function renderWithMocks(mocks: MockedResponse[]) { + return renderHook(() => useContext(WorkspaceContext), { + wrapper({children}) { + return ( + + + + + {children} + + + + + ); + }, + }); +} + +function getLocationMocks(updatedTimestamp = 0) { + const repositoryLocation1 = buildRepositoryLocation({ + repositories: [buildRepository(), buildRepository()], + }); + const repositoryLocation2 = buildRepositoryLocation({ + repositories: [buildRepository(), buildRepository()], + }); + const repositoryLocation3 = buildRepositoryLocation({ + repositories: [buildRepository(), buildRepository()], + }); + const location1 = buildWorkspaceLocationEntry({ + name: 'location1', + updatedTimestamp, + locationOrLoadError: repositoryLocation1, + }); + const location2 = buildWorkspaceLocationEntry({ + name: 'location2', + updatedTimestamp, + locationOrLoadError: repositoryLocation2, + }); + + const location3 = buildWorkspaceLocationEntry({ + name: 'location3', + updatedTimestamp, + locationOrLoadError: repositoryLocation3, + }); + + return { + repositoryLocation1, + repositoryLocation2, + repositoryLocation3, + location1, + location2, + location3, + caches: { + codeLocationStatusQuery: mockCache({ + dbName: `${KEY_PREFIX}${LOCAL_CACHE_ID_PREFIX}${CODE_LOCATION_STATUS_QUERY_KEY}`, + }), + location1: mockCache({ + dbName: `${KEY_PREFIX}${LOCAL_CACHE_ID_PREFIX}${locationWorkspaceKey('location1')}`, + }), + location2: mockCache({ + dbName: `${KEY_PREFIX}${LOCAL_CACHE_ID_PREFIX}${locationWorkspaceKey('location2')}`, + }), + location3: mockCache({ + dbName: `${KEY_PREFIX}${LOCAL_CACHE_ID_PREFIX}${locationWorkspaceKey('location3')}`, + }), + }, + }; +} + +describe('WorkspaceContext', () => { + it('Fetches by code location when cache is empty', async () => { + const { + location1, + location2, + location3, + repositoryLocation1, + repositoryLocation2, + repositoryLocation3, + caches, + } = getLocationMocks(-1); + caches.codeLocationStatusQuery.has.mockResolvedValue(false); + caches.location1.has.mockResolvedValue(false); + caches.location2.has.mockResolvedValue(false); + caches.location3.has.mockResolvedValue(false); + const mocks = buildWorkspaceMocks([location1, location2, location3], {delay: 10}); + const mockCbs = mocks.map(getMockResultFn); + + // Include code location status mock a second time since we call runOnlyPendingTimersAsync twice + const {result} = renderWithMocks([...mocks, mocks[0]!]); + + expect(result.current.allRepos).toEqual([]); + expect(result.current.data).toEqual({}); + expect(result.current.loading).toEqual(true); + + // Runs the code location status query + await act(async () => { + await jest.runOnlyPendingTimersAsync(); + }); + + // First mock is the code location status query + expect(mockCbs[0]).toHaveBeenCalled(); + expect(mockCbs[1]).not.toHaveBeenCalled(); + expect(mockCbs[2]).not.toHaveBeenCalled(); + expect(mockCbs[3]).not.toHaveBeenCalled(); + + expect(result.current.allRepos).toEqual([]); + expect(result.current.data).toEqual({}); + expect(result.current.loading).toEqual(true); + + // Runs the individual location queries + await act(async () => { + await jest.runOnlyPendingTimersAsync(); + }); + + expect(mockCbs[1]).toHaveBeenCalled(); + expect(mockCbs[2]).toHaveBeenCalled(); + expect(mockCbs[3]).toHaveBeenCalled(); + + await waitFor(() => { + expect(result.current.loading).toEqual(false); + }); + expect(result.current.allRepos).toEqual([ + ...repoLocationToRepos(repositoryLocation1), + ...repoLocationToRepos(repositoryLocation2), + ...repoLocationToRepos(repositoryLocation3), + ]); + expect(result.current.data).toEqual({ + [location1.name]: location1, + [location2.name]: location2, + [location3.name]: location3, + }); + + await act(async () => { + await jest.runAllTicks(); + }); + }); + + it('Uses cache if it is up to date and does not query by location', async () => { + const { + location1, + location2, + location3, + repositoryLocation1, + repositoryLocation2, + repositoryLocation3, + caches, + } = getLocationMocks(); + const mocks = buildWorkspaceMocks([location1, location2, location3], {delay: 10}); + + caches.codeLocationStatusQuery.has.mockResolvedValue(true); + caches.codeLocationStatusQuery.get.mockResolvedValue({ + value: {data: (mocks[0]! as any).result.data, version: CODE_LOCATION_STATUS_QUERY_VERSION}, + }); + caches.location1.has.mockResolvedValue(true); + caches.location1.get.mockResolvedValue({ + value: {data: (mocks[1]! as any).result.data, version: LOCATION_WORKSPACE_QUERY_VERSION}, + }); + caches.location2.has.mockResolvedValue(true); + caches.location2.get.mockResolvedValue({ + value: {data: (mocks[2]! as any).result.data, version: LOCATION_WORKSPACE_QUERY_VERSION}, + }); + caches.location3.has.mockResolvedValue(true); + caches.location3.get.mockResolvedValue({ + value: {data: (mocks[3]! as any).result.data, version: LOCATION_WORKSPACE_QUERY_VERSION}, + }); + + const mockCbs = mocks.map(getMockResultFn); + + const {result} = renderWithMocks([...mocks]); + + // await act(async () => { + // await jest.runOnlyPendingTimersAsync(); + // }); + + await waitFor(async () => { + expect(result.current.loading).toEqual(false); + }); + // We queries for code location statuses but saw we were up to date + // so we didn't call any the location queries + expect(mockCbs[0]).toHaveBeenCalled(); + + expect(mockCbs[1]).not.toHaveBeenCalled(); + expect(mockCbs[2]).not.toHaveBeenCalled(); + expect(mockCbs[3]).not.toHaveBeenCalled(); + expect(result.current.allRepos).toEqual([ + ...repoLocationToRepos(repositoryLocation1), + ...repoLocationToRepos(repositoryLocation2), + ...repoLocationToRepos(repositoryLocation3), + ]); + expect(result.current.data).toEqual({ + [location1.name]: location1, + [location2.name]: location2, + [location3.name]: location3, + }); + + await act(async () => { + await jest.runAllTicks(); + }); + }); + + it('returns cached data, detects its out of date and queries by location to update it', async () => { + const { + location1, + location2, + location3, + repositoryLocation1, + repositoryLocation2, + repositoryLocation3, + caches, + } = getLocationMocks(-3); + const mocks = buildWorkspaceMocks([location1, location2, location3], {delay: 10}); + const { + location1: updatedLocation1, + location2: updatedLocation2, + location3: updatedLocation3, + } = getLocationMocks(1); + + const updatedMocks = buildWorkspaceMocks( + [updatedLocation1, updatedLocation2, updatedLocation3], + {delay: 10}, + ); + + caches.codeLocationStatusQuery.has.mockResolvedValue(true); + caches.codeLocationStatusQuery.get.mockResolvedValue({ + value: {data: (mocks[0]! as any).result.data, version: CODE_LOCATION_STATUS_QUERY_VERSION}, + }); + caches.location1.has.mockResolvedValue(true); + caches.location1.get.mockResolvedValue({ + value: {data: (mocks[1]! as any).result.data, version: LOCATION_WORKSPACE_QUERY_VERSION}, + }); + caches.location2.has.mockResolvedValue(true); + caches.location2.get.mockResolvedValue({ + value: {data: (mocks[2]! as any).result.data, version: LOCATION_WORKSPACE_QUERY_VERSION}, + }); + caches.location3.has.mockResolvedValue(true); + caches.location3.get.mockResolvedValue({ + value: {data: (mocks[3]! as any).result.data, version: LOCATION_WORKSPACE_QUERY_VERSION}, + }); + const mockCbs = updatedMocks.map(getMockResultFn); + + const {result} = renderWithMocks([...updatedMocks, updatedMocks[0]!]); + + await act(async () => { + await jest.runOnlyPendingTimersAsync(); + }); + // We queries for code location statuses and we see we are not up to date so we call the location queries + // but their data hasn't returned yet so the current data is still equal to the cached data + expect(mockCbs[0]).toHaveBeenCalled(); + expect(mockCbs[1]).not.toHaveBeenCalled(); + expect(mockCbs[2]).not.toHaveBeenCalled(); + expect(mockCbs[3]).not.toHaveBeenCalled(); + + expect(result.current.allRepos).toEqual([ + ...repoLocationToRepos(repositoryLocation1), + ...repoLocationToRepos(repositoryLocation2), + ...repoLocationToRepos(repositoryLocation3), + ]); + expect(result.current.data).toEqual({ + [location1.name]: location1, + [location2.name]: location2, + [location3.name]: location3, + }); + + // Wait for the location queries to return + await act(async () => { + await jest.runOnlyPendingTimersAsync(); + }); + + expect(result.current.data).toEqual({ + [location1.name]: updatedLocation1, + [location2.name]: updatedLocation2, + [location3.name]: updatedLocation3, + }); + expect(updatedLocation1).not.toEqual(location1); + + await act(async () => { + await jest.runAllTicks(); + }); + }); + + it('Detects and deletes removed code locations from cache', async () => { + const { + location1, + location2, + location3, + repositoryLocation1, + repositoryLocation2, + repositoryLocation3, + caches, + } = getLocationMocks(0); + const mocks = buildWorkspaceMocks([location1, location2, location3], {delay: 10}); + const {location1: updatedLocation1, location2: updatedLocation2} = getLocationMocks(1); + + // Remove location 3 + const updatedMocks = buildWorkspaceMocks([updatedLocation1, updatedLocation2], {delay: 10}); + + caches.codeLocationStatusQuery.has.mockResolvedValue(true); + caches.codeLocationStatusQuery.get.mockResolvedValue({ + value: {data: (mocks[0]! as any).result.data, version: CODE_LOCATION_STATUS_QUERY_VERSION}, + }); + caches.location1.has.mockResolvedValue(true); + caches.location1.get.mockResolvedValue({ + value: {data: (mocks[1]! as any).result.data, version: LOCATION_WORKSPACE_QUERY_VERSION}, + }); + caches.location2.has.mockResolvedValue(true); + caches.location2.get.mockResolvedValue({ + value: {data: (mocks[2]! as any).result.data, version: LOCATION_WORKSPACE_QUERY_VERSION}, + }); + caches.location3.has.mockResolvedValue(true); + caches.location3.get.mockResolvedValue({ + value: {data: (mocks[3]! as any).result.data, version: LOCATION_WORKSPACE_QUERY_VERSION}, + }); + const mockCbs = updatedMocks.map(getMockResultFn); + + const {result} = renderWithMocks([...updatedMocks, updatedMocks[0]!]); + + await act(async () => { + await jest.runAllTicks(); + }); + + await waitFor(async () => { + expect(result.current.loading).toEqual(false); + }); + + // We return the cached data + expect(result.current.allRepos).toEqual([ + ...repoLocationToRepos(repositoryLocation1), + ...repoLocationToRepos(repositoryLocation2), + ...repoLocationToRepos(repositoryLocation3), + ]); + + expect(result.current.data).toEqual({ + [location1.name]: location1, + [location2.name]: location2, + [location3.name]: location3, + }); + + expect(mockCbs[0]).toHaveBeenCalled(); + expect(mockCbs[1]).not.toHaveBeenCalled(); + expect(mockCbs[2]).not.toHaveBeenCalled(); + + await act(async () => { + // Our mocks have a delay of 10 + jest.advanceTimersByTime(10); + jest.runAllTicks(); + }); + + // We detect the third code location was deleted + expect(result.current.allRepos).toEqual([ + ...repoLocationToRepos(repositoryLocation1), + ...repoLocationToRepos(repositoryLocation2), + ]); + + expect(result.current.data).toEqual({ + [location1.name]: location1, + [location2.name]: location2, + }); + + // We query for the the updated cached locations + await act(async () => { + await jest.runOnlyPendingTimersAsync(); + }); + + // We now have the latest data + expect(result.current.data).toEqual({ + [location1.name]: updatedLocation1, + [location2.name]: updatedLocation2, + }); + }); +}); From 4d7af4c3d632f4be35bf5063232c56bbc2a5671a Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Thu, 6 Jun 2024 23:35:55 -0400 Subject: [PATCH 10/16] comment --- .../src/workspace/__tests__/WorkspaceContext.test.tsx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/__tests__/WorkspaceContext.test.tsx b/js_modules/dagster-ui/packages/ui-core/src/workspace/__tests__/WorkspaceContext.test.tsx index 51198496288b8..67c998bd64c6b 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/__tests__/WorkspaceContext.test.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/__tests__/WorkspaceContext.test.tsx @@ -194,6 +194,7 @@ describe('WorkspaceContext', () => { }); await act(async () => { + // Exhaust any remaining tasks so they don't affect the next test. await jest.runAllTicks(); }); }); @@ -257,6 +258,7 @@ describe('WorkspaceContext', () => { }); await act(async () => { + // Exhaust any remaining tasks so they don't affect the next test. await jest.runAllTicks(); }); }); @@ -306,8 +308,7 @@ describe('WorkspaceContext', () => { await act(async () => { await jest.runOnlyPendingTimersAsync(); }); - // We queries for code location statuses and we see we are not up to date so we call the location queries - // but their data hasn't returned yet so the current data is still equal to the cached data + // We queries for code location statuses and we see we are not up to date yet so the current data is still equal to the cached data expect(mockCbs[0]).toHaveBeenCalled(); expect(mockCbs[1]).not.toHaveBeenCalled(); expect(mockCbs[2]).not.toHaveBeenCalled(); @@ -324,7 +325,7 @@ describe('WorkspaceContext', () => { [location3.name]: location3, }); - // Wait for the location queries to return + // Run the location queries and wait for the location queries to return await act(async () => { await jest.runOnlyPendingTimersAsync(); }); @@ -337,6 +338,7 @@ describe('WorkspaceContext', () => { expect(updatedLocation1).not.toEqual(location1); await act(async () => { + // Exhaust any remaining tasks so they don't affect the next test. await jest.runAllTicks(); }); }); From ec731a7b05ee8e60b0ec881da7b8bc836cf2ce52 Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Thu, 6 Jun 2024 23:59:25 -0400 Subject: [PATCH 11/16] more defensive... --- .../src/workspace/WorkspaceContext.tsx | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx index 70e03773fb35e..e7daaa4f9377b 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx @@ -234,17 +234,18 @@ export const WorkspaceProvider = ({children}: {children: React.ReactNode}) => { ); useLayoutEffect(() => { - if (locationsRemoved.length) { - setLocationsData((locationsData) => { - const copy = {...locationsData}; - locationsRemoved.forEach((loc) => { - delete copy[loc.name]; - indexedDB.deleteDatabase(`${localCacheIdPrefix}${locationWorkspaceKey(loc.name)}`); - }); - return copy; - }); + if (!locationsRemoved.length) { + return; + } + const copy = {...locationsData}; + locationsRemoved.forEach((loc) => { + delete copy[loc.name]; + indexedDB.deleteDatabase(`${localCacheIdPrefix}${locationWorkspaceKey(loc.name)}`); + }); + if (Object.keys(copy).length !== Object.keys(locationsData).length) { + setLocationsData(copy); } - }, [localCacheIdPrefix, locationsRemoved]); + }, [localCacheIdPrefix, locationsData, locationsRemoved]); const locationEntries = useMemo( () => From 2afd1780b129d08a8a3cca3e059b9d9b73067397 Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Fri, 7 Jun 2024 00:15:54 -0400 Subject: [PATCH 12/16] . --- .../dagster-ui/packages/ui-core/src/code-links/CodeLink.tsx | 3 +-- .../packages/ui-core/src/workspace/WorkspaceContext.tsx | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/js_modules/dagster-ui/packages/ui-core/src/code-links/CodeLink.tsx b/js_modules/dagster-ui/packages/ui-core/src/code-links/CodeLink.tsx index d723df1643383..35ff702b6b8b9 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/code-links/CodeLink.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/code-links/CodeLink.tsx @@ -1,5 +1,4 @@ -import {Box, MiddleTruncate, Tooltip} from '@dagster-io/ui-components'; -import {Icon, IconName} from '@dagster-io/ui-components/src/components/Icon'; +import {Box, Icon, IconName, MiddleTruncate, Tooltip} from '@dagster-io/ui-components'; import * as React from 'react'; import {CodeLinkProtocolContext, ProtocolData} from './CodeLinkProtocol'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx index e7daaa4f9377b..2bc857cb55549 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx @@ -189,7 +189,7 @@ export const WorkspaceProvider = ({children}: {children: React.ReactNode}) => { ); return locationData; }, - [client, localCacheIdPrefix], + [client, getData, localCacheIdPrefix], ); const locationsToFetch = useMemo(() => { From 2d1879d15d506e5700adba863f7f6a92c06a8f11 Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Fri, 7 Jun 2024 00:55:06 -0400 Subject: [PATCH 13/16] fix toast --- .../src/nav/useCodeLocationsStatus.tsx | 32 ++++---- .../src/workspace/CodeLocationRowSet.tsx | 82 ------------------- .../src/workspace/WorkspaceContext.tsx | 17 +++- 3 files changed, 30 insertions(+), 101 deletions(-) diff --git a/js_modules/dagster-ui/packages/ui-core/src/nav/useCodeLocationsStatus.tsx b/js_modules/dagster-ui/packages/ui-core/src/nav/useCodeLocationsStatus.tsx index 2f276e0fc95e3..1a3f0db69db01 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/nav/useCodeLocationsStatus.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/nav/useCodeLocationsStatus.tsx @@ -1,5 +1,5 @@ import {Box, ButtonLink, Colors} from '@dagster-io/ui-components'; -import {useCallback, useContext, useEffect, useLayoutEffect, useState} from 'react'; +import {useCallback, useContext, useLayoutEffect, useState} from 'react'; import {useHistory} from 'react-router-dom'; import {atom, useRecoilValue} from 'recoil'; import styled from 'styled-components'; @@ -38,8 +38,7 @@ export const useCodeLocationsStatus = (): StatusAndMessage | null => { // Reload the workspace, but don't toast about it. // Reload the workspace, and show a success or error toast upon completion. - useEffect(() => { - const isFreshPageload = previousEntriesById === null; + useLayoutEffect(() => { const anyErrors = Object.values(data).some( (entry) => entry.__typename === 'PythonError' || @@ -59,17 +58,6 @@ export const useCodeLocationsStatus = (): StatusAndMessage | null => { ), icon: 'check_circle', }); - } else if (!isFreshPageload) { - showSharedToaster({ - intent: 'success', - message: ( - -
Definitions reloaded
- {showViewButton ? : null} - - ), - icon: 'check_circle', - }); } // eslint-disable-next-line react-hooks/exhaustive-deps }, [data, onClickViewButton]); @@ -78,6 +66,7 @@ export const useCodeLocationsStatus = (): StatusAndMessage | null => { useLayoutEffect(() => { const isFreshPageload = previousEntriesById === null; + const showViewButton = !alreadyViewingCodeLocations(); // Given the previous and current code locations, determine whether to show a) a loading spinner // and/or b) a toast indicating that a code location is being reloaded. @@ -88,6 +77,19 @@ export const useCodeLocationsStatus = (): StatusAndMessage | null => { : []; let hasUpdatedEntries = entries.length !== Object.keys(previousEntriesById || {}).length; + + if (!isFreshPageload && hasUpdatedEntries) { + showSharedToaster({ + intent: 'success', + message: ( + +
Definitions reloaded
+ {showViewButton ? : null} + + ), + icon: 'check_circle', + }); + } const currEntriesById: {[key: string]: LocationStatusEntry} = {}; entries.forEach((entry) => { const previousEntry = previousEntriesById && previousEntriesById[entry.id]; @@ -130,8 +132,6 @@ export const useCodeLocationsStatus = (): StatusAndMessage | null => { return; } - const showViewButton = !alreadyViewingCodeLocations(); - // We have a new entry, and it has already finished loading. Wow! It's surprisingly fast for it // to have finished loading so quickly, but go ahead and indicate that the location has // been added, then reload the workspace. diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/CodeLocationRowSet.tsx b/js_modules/dagster-ui/packages/ui-core/src/workspace/CodeLocationRowSet.tsx index a03837c974bf4..0f684461839a6 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/CodeLocationRowSet.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/CodeLocationRowSet.tsx @@ -4,23 +4,16 @@ import { ButtonLink, Colors, Icon, - JoinedButtons, MiddleTruncate, Tag, Tooltip, } from '@dagster-io/ui-components'; import {useCallback, useMemo, useState} from 'react'; -import {Link} from 'react-router-dom'; import styled from 'styled-components'; -import {CodeLocationMenu} from './CodeLocationMenu'; -import {RepositoryCountTags} from './RepositoryCountTags'; import {RepositoryLocationNonBlockingErrorDialog} from './RepositoryLocationErrorDialog'; import {WorkspaceRepositoryLocationNode} from './WorkspaceContext'; -import {buildRepoAddress} from './buildRepoAddress'; -import {repoAddressAsHumanString} from './repoAddressAsString'; import {WorkspaceDisplayMetadataFragment} from './types/WorkspaceQueries.types'; -import {workspacePathFromAddress} from './workspacePath'; import {showSharedToaster} from '../app/DomUtils'; import {useCopyToClipboard} from '../app/browser'; import { @@ -31,81 +24,6 @@ import { buildReloadFnForLocation, useRepositoryLocationReload, } from '../nav/useRepositoryLocationReload'; -import {TimeFromNow} from '../ui/TimeFromNow'; - -interface Props { - locationNode: WorkspaceRepositoryLocationNode; -} - -export const CodeLocationRowSet = ({locationNode}: Props) => { - const {name, locationOrLoadError} = locationNode; - - if (!locationOrLoadError || locationOrLoadError?.__typename === 'PythonError') { - return ( - - - - - - - - - - - {'\u2013'} - - - - - - - - ); - } - - const repositories = [...locationOrLoadError.repositories].sort((a, b) => - a.name.localeCompare(b.name), - ); - - return ( - <> - {repositories.map((repository) => { - const repoAddress = buildRepoAddress(repository.name, name); - const allMetadata = [...locationNode.displayMetadata, ...repository.displayMetadata]; - return ( - - - -
- - - -
- - - - - - - - - - - - - - - - - - - - - ); - })} - - ); -}; export const ImageName = ({metadata}: {metadata: WorkspaceDisplayMetadataFragment[]}) => { const copy = useCopyToClipboard(); diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx index 2bc857cb55549..a2f2119f57109 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx @@ -192,19 +192,30 @@ export const WorkspaceProvider = ({children}: {children: React.ReactNode}) => { [client, getData, localCacheIdPrefix], ); + const [isRefetching, setIsRefetching] = useState(false); + const locationsToFetch = useMemo(() => { if (!didLoadCachedData) { return []; } + if (isRefetching) { + return []; + } const toFetch = Object.values(locations).filter((loc) => { const prev = prevLocations.current?.[loc.name]; - return prev?.updateTimestamp !== loc.updateTimestamp || prev?.loadStatus !== loc.loadStatus; + const d = locationsData[loc.name]; + const entry = d?.__typename === 'WorkspaceLocationEntry' ? d : null; + return ( + prev?.updateTimestamp !== loc.updateTimestamp || + prev?.loadStatus !== loc.loadStatus || + entry?.updatedTimestamp !== loc.updateTimestamp || + entry?.loadStatus !== loc.loadStatus + ); }); prevLocations.current = locations; return toFetch; - }, [locations, didLoadCachedData]); + }, [didLoadCachedData, isRefetching, locations, locationsData]); - const [isRefetching, setIsRefetching] = useState(false); useLayoutEffect(() => { if (!locationsToFetch.length) { return; From 552d1eca149a398eb7105a8f8fa3b224a46ee4f3 Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Fri, 7 Jun 2024 04:30:22 -0400 Subject: [PATCH 14/16] . --- .../packages/ui-core/src/workspace/WorkspaceContext.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx index a2f2119f57109..332f924139185 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx @@ -208,7 +208,6 @@ export const WorkspaceProvider = ({children}: {children: React.ReactNode}) => { return ( prev?.updateTimestamp !== loc.updateTimestamp || prev?.loadStatus !== loc.loadStatus || - entry?.updatedTimestamp !== loc.updateTimestamp || entry?.loadStatus !== loc.loadStatus ); }); From 4c01aea63ebe3346b5d5fc7aa5e5a926d96b33e6 Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Sun, 9 Jun 2024 21:45:22 -0400 Subject: [PATCH 15/16] clear cache data --- .../src/search/useIndexedDBCachedQuery.tsx | 16 +++++++++++++++- .../ui-core/src/workspace/WorkspaceContext.tsx | 6 ++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/js_modules/dagster-ui/packages/ui-core/src/search/useIndexedDBCachedQuery.tsx b/js_modules/dagster-ui/packages/ui-core/src/search/useIndexedDBCachedQuery.tsx index d68344bf138c5..16d016d6b9804 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/search/useIndexedDBCachedQuery.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/search/useIndexedDBCachedQuery.tsx @@ -29,9 +29,13 @@ export class CacheManager { return null; } - set(data: TQuery, version: number): Promise { + async set(data: TQuery, version: number): Promise { return this.cache.set('cache', {data, version}, {expiry: new Date('3030-01-01')}); } + + async clear() { + await this.cache.delete('cache'); + } } interface QueryHookParams { @@ -158,6 +162,16 @@ export function useGetCachedData() { [], ); } +export function useClearCachedData() { + const {getCacheManager} = useContext(IndexedDBCacheContext); + return useCallback( + async ({key}: {key: string}) => { + const cacheManager = getCacheManager(key); + await cacheManager.clear(); + }, + [getCacheManager], + ); +} const contextValue = createIndexedDBCacheContextValue(); export const IndexedDBCacheContext = createContext(contextValue); diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx index 332f924139185..57c69bd6be8c0 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceContext.tsx @@ -26,6 +26,7 @@ import {useStateWithStorage} from '../hooks/useStateWithStorage'; import {useUpdatingRef} from '../hooks/useUpdatingRef'; import {codeLocationStatusAtom} from '../nav/useCodeLocationsStatus'; import { + useClearCachedData, useGetCachedData, useGetData, useIndexedDBCachedQuery, @@ -114,6 +115,7 @@ export const WorkspaceProvider = ({children}: {children: React.ReactNode}) => { const getCachedData = useGetCachedData(); const getData = useGetData(); + const clearCachedData = useClearCachedData(); useLayoutEffect(() => { // Load data from the cache @@ -250,12 +252,12 @@ export const WorkspaceProvider = ({children}: {children: React.ReactNode}) => { const copy = {...locationsData}; locationsRemoved.forEach((loc) => { delete copy[loc.name]; - indexedDB.deleteDatabase(`${localCacheIdPrefix}${locationWorkspaceKey(loc.name)}`); + clearCachedData({key: `${localCacheIdPrefix}${locationWorkspaceKey(loc.name)}`}); }); if (Object.keys(copy).length !== Object.keys(locationsData).length) { setLocationsData(copy); } - }, [localCacheIdPrefix, locationsData, locationsRemoved]); + }, [clearCachedData, localCacheIdPrefix, locationsData, locationsRemoved]); const locationEntries = useMemo( () => From 2997ccd87476baaae32700ac88a62adf44d0d379 Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Sun, 9 Jun 2024 22:02:14 -0400 Subject: [PATCH 16/16] test mock --- .../src/search/__tests__/useIndexedDBCachedQuery.test.tsx | 1 + .../ui-core/src/workspace/__tests__/WorkspaceContext.test.tsx | 1 + 2 files changed, 2 insertions(+) diff --git a/js_modules/dagster-ui/packages/ui-core/src/search/__tests__/useIndexedDBCachedQuery.test.tsx b/js_modules/dagster-ui/packages/ui-core/src/search/__tests__/useIndexedDBCachedQuery.test.tsx index 2529134c9ab4d..747099649b26b 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/search/__tests__/useIndexedDBCachedQuery.test.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/search/__tests__/useIndexedDBCachedQuery.test.tsx @@ -24,6 +24,7 @@ jest.mock('idb-lru-cache', () => { has: jest.fn(), get: jest.fn(), set: jest.fn(), + delete: jest.fn(), }; return { cache: jest.fn(() => mockedCache), diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/__tests__/WorkspaceContext.test.tsx b/js_modules/dagster-ui/packages/ui-core/src/workspace/__tests__/WorkspaceContext.test.tsx index 67c998bd64c6b..e0d47c18667ac 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/__tests__/WorkspaceContext.test.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/__tests__/WorkspaceContext.test.tsx @@ -41,6 +41,7 @@ jest.mock('idb-lru-cache', () => { has: jest.fn(), get: jest.fn(), set: jest.fn(), + delete: jest.fn(), }; } return mockedCacheStore[dbName];