From f5f965a3e15b6a0887a41a9bcd313ba367888f18 Mon Sep 17 00:00:00 2001 From: Isaac Hellendag Date: Mon, 19 Aug 2024 13:10:36 -0500 Subject: [PATCH] [ui] Code location: Graphs page --- .../CodeLocationDefinitionsMain.tsx | 4 + .../code-location/CodeLocationGraphsList.tsx | 113 ++++++++++++++++++ .../CodeLocationPages.fixtures.ts | 41 +++++++ .../CodeLocationDefinitionsRoot.stories.tsx | 12 +- .../src/workspace/WorkspaceGraphsQuery.tsx | 48 ++++++++ .../src/workspace/WorkspaceGraphsRoot.tsx | 74 +----------- .../src/workspace/extractGraphsForRepo.tsx | 31 +++++ ...types.ts => WorkspaceGraphsQuery.types.ts} | 28 +++++ 8 files changed, 282 insertions(+), 69 deletions(-) create mode 100644 js_modules/dagster-ui/packages/ui-core/src/code-location/CodeLocationGraphsList.tsx create mode 100644 js_modules/dagster-ui/packages/ui-core/src/workspace/WorkspaceGraphsQuery.tsx create mode 100644 js_modules/dagster-ui/packages/ui-core/src/workspace/extractGraphsForRepo.tsx rename js_modules/dagster-ui/packages/ui-core/src/workspace/types/{WorkspaceGraphsRoot.types.ts => WorkspaceGraphsQuery.types.ts} (67%) diff --git a/js_modules/dagster-ui/packages/ui-core/src/code-location/CodeLocationDefinitionsMain.tsx b/js_modules/dagster-ui/packages/ui-core/src/code-location/CodeLocationDefinitionsMain.tsx index 3a8a05c95f55a..8ce28a98a1c91 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/code-location/CodeLocationDefinitionsMain.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/code-location/CodeLocationDefinitionsMain.tsx @@ -2,6 +2,7 @@ import {Box} from '@dagster-io/ui-components'; import {useMemo} from 'react'; import {Switch} from 'react-router-dom'; +import {CodeLocationGraphsList} from './CodeLocationGraphsList'; import {CodeLocationOpsView} from './CodeLocationOpsView'; import {CodeLocationSearchableList, SearchableListRow} from './CodeLocationSearchableList'; import {Route} from '../app/Route'; @@ -32,6 +33,9 @@ export const CodeLocationDefinitionsMain = ({repoAddress, repository}: Props) => + + + diff --git a/js_modules/dagster-ui/packages/ui-core/src/code-location/CodeLocationGraphsList.tsx b/js_modules/dagster-ui/packages/ui-core/src/code-location/CodeLocationGraphsList.tsx new file mode 100644 index 0000000000000..7ae38a31c8e5b --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/code-location/CodeLocationGraphsList.tsx @@ -0,0 +1,113 @@ +import {useQuery} from '@apollo/client'; +import {Box, NonIdealState, SpinnerWithText} from '@dagster-io/ui-components'; +import {useMemo} from 'react'; + +import {CodeLocationSearchableList, SearchableListRow} from './CodeLocationSearchableList'; +import {PythonErrorInfo} from '../app/PythonErrorInfo'; +import {useBlockTraceOnQueryResult} from '../performance/TraceContext'; +import {WORSKPACE_GRAPHS_QUERY} from '../workspace/WorkspaceGraphsQuery'; +import {extractGraphsForRepo} from '../workspace/extractGraphsForRepo'; +import {repoAddressAsHumanString} from '../workspace/repoAddressAsString'; +import {repoAddressToSelector} from '../workspace/repoAddressToSelector'; +import {RepoAddress} from '../workspace/types'; +import { + WorkspaceGraphsQuery, + WorkspaceGraphsQueryVariables, +} from '../workspace/types/WorkspaceGraphsQuery.types'; +import {workspacePathFromAddress} from '../workspace/workspacePath'; + +interface Props { + repoAddress: RepoAddress; +} + +export const CodeLocationGraphsList = (props: Props) => { + const {repoAddress} = props; + + const selector = repoAddressToSelector(repoAddress); + + const queryResult = useQuery( + WORSKPACE_GRAPHS_QUERY, + {variables: {selector}}, + ); + + useBlockTraceOnQueryResult(queryResult, 'WorkspaceGraphsQuery'); + const {data, loading} = queryResult; + + const graphs = useMemo(() => { + const repo = data?.repositoryOrError; + if (!repo || repo.__typename !== 'Repository') { + return []; + } + + return extractGraphsForRepo(repo); + }, [data]); + + const repoString = repoAddressAsHumanString(repoAddress); + + if (loading) { + return ( + + + + ); + } + + if (!data || !data.repositoryOrError) { + return ( + + + + ); + } + + if (data.repositoryOrError.__typename === 'PythonError') { + return ( + + + + ); + } + + if (data.repositoryOrError.__typename === 'RepositoryNotFoundError') { + return ( + + + + ); + } + + if (!graphs.length) { + return ( + + + + ); + } + + return ( + graph.name.toLowerCase().includes(value)} + renderRow={(graph) => ( + + )} + /> + ); +}; diff --git a/js_modules/dagster-ui/packages/ui-core/src/code-location/__fixtures__/CodeLocationPages.fixtures.ts b/js_modules/dagster-ui/packages/ui-core/src/code-location/__fixtures__/CodeLocationPages.fixtures.ts index 7446a0c2abdc7..f968983d7e0c0 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/code-location/__fixtures__/CodeLocationPages.fixtures.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/code-location/__fixtures__/CodeLocationPages.fixtures.ts @@ -1,6 +1,7 @@ import faker from 'faker'; import { + buildCompositeSolidDefinition, buildDagsterLibraryVersion, buildPipeline, buildRepository, @@ -16,8 +17,13 @@ import { import {OPS_ROOT_QUERY} from '../../ops/OpsRoot'; import {OpsRootQuery, OpsRootQueryVariables} from '../../ops/types/OpsRoot.types'; import {buildQueryMock} from '../../testing/mocking'; +import {WORSKPACE_GRAPHS_QUERY} from '../../workspace/WorkspaceGraphsQuery'; import {repoAddressToSelector} from '../../workspace/repoAddressToSelector'; import {RepoAddress} from '../../workspace/types'; +import { + WorkspaceGraphsQuery, + WorkspaceGraphsQueryVariables, +} from '../../workspace/types/WorkspaceGraphsQuery.types'; export const buildEmptyWorkspaceLocationEntry = (config: {time: number; locationName: string}) => { const {time, locationName} = config; @@ -107,5 +113,40 @@ export const buildSampleOpsRootQuery = (config: {repoAddress: RepoAddress; opCou }), }, delay: 2000, + maxUsageCount: 20, + }); +}; + +export const buildSampleRepositoryGraphsQuery = (config: { + repoAddress: RepoAddress; + jobCount: number; + opCount: number; +}) => { + const {repoAddress, jobCount, opCount} = config; + return buildQueryMock({ + query: WORSKPACE_GRAPHS_QUERY, + variables: { + selector: repoAddressToSelector(repoAddress), + }, + data: { + repositoryOrError: buildRepository({ + usedSolids: new Array(opCount).fill(null).map(() => { + return buildUsedSolid({ + definition: buildCompositeSolidDefinition({ + name: faker.random.words(2).split(' ').join('-').toLowerCase(), + }), + }); + }), + pipelines: new Array(jobCount).fill(null).map(() => { + return buildPipeline({ + id: faker.datatype.uuid(), + graphName: faker.random.words(2).split(' ').join('-').toLowerCase(), + isJob: true, + }); + }), + }), + }, + delay: 2000, + maxUsageCount: 100, }); }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/code-location/__stories__/CodeLocationDefinitionsRoot.stories.tsx b/js_modules/dagster-ui/packages/ui-core/src/code-location/__stories__/CodeLocationDefinitionsRoot.stories.tsx index 6f6d265275cb0..7f705f58aad80 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/code-location/__stories__/CodeLocationDefinitionsRoot.stories.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/code-location/__stories__/CodeLocationDefinitionsRoot.stories.tsx @@ -1,5 +1,6 @@ import {MockedProvider} from '@apollo/client/testing'; import {Meta} from '@storybook/react'; +import {useMemo} from 'react'; import {MemoryRouter} from 'react-router'; import {RecoilRoot} from 'recoil'; @@ -9,6 +10,7 @@ import {CodeLocationDefinitionsRoot} from '../CodeLocationDefinitionsRoot'; import { buildSampleOpsRootQuery, buildSampleRepository, + buildSampleRepositoryGraphsQuery, } from '../__fixtures__/CodeLocationPages.fixtures'; // eslint-disable-next-line import/no-default-export @@ -31,10 +33,18 @@ export const Default = () => { resourceCount: 100, }); + const mocks = useMemo( + () => [ + buildSampleOpsRootQuery({repoAddress, opCount: 500}), + buildSampleRepositoryGraphsQuery({repoAddress, jobCount: 500, opCount: 500}), + ], + [repoAddress], + ); + return ( - +
( - repo.pipelines - .filter((p) => p.isJob && !isHiddenAssetGroupJob(p.name)) - .map((p) => p.graphName), - ); - - const items: Graph[] = Array.from(jobGraphNames).map((graphName) => ({ - name: graphName, - path: `/graphs/${graphName}`, - description: null, - })); - - repo.usedSolids.forEach((s) => { - if (s.definition.__typename === 'CompositeSolidDefinition') { - const invocation = s.invocations[0]; - if (invocation) { - items.push({ - name: s.definition.name, - path: `/graphs/${invocation.pipeline.name}/${invocation.solidHandle.handleID}/`, - description: s.definition.description, - }); - } - } - }); - - return items.sort((a, b) => a.name.localeCompare(b.name)); + return extractGraphsForRepo(repo); }, [data]); const filteredBySearch = useMemo(() => { @@ -150,40 +125,3 @@ export const WorkspaceGraphsRoot = ({repoAddress}: {repoAddress: RepoAddress}) = ); }; - -const WORSKPACE_GRAPHS_QUERY = gql` - query WorkspaceGraphsQuery($selector: RepositorySelector!) { - repositoryOrError(repositorySelector: $selector) { - ... on Repository { - id - usedSolids { - definition { - ... on CompositeSolidDefinition { - id - name - description - } - } - invocations { - pipeline { - id - name - } - solidHandle { - handleID - } - } - } - pipelines { - id - name - isJob - graphName - } - } - ...PythonErrorFragment - } - } - - ${PYTHON_ERROR_FRAGMENT} -`; diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/extractGraphsForRepo.tsx b/js_modules/dagster-ui/packages/ui-core/src/workspace/extractGraphsForRepo.tsx new file mode 100644 index 0000000000000..f849d48390f2f --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/extractGraphsForRepo.tsx @@ -0,0 +1,31 @@ +import {Graph} from './VirtualizedGraphTable'; +import {RepositoryGraphsFragment} from './types/WorkspaceGraphsQuery.types'; +import {COMMON_COLLATOR} from '../app/Util'; +import {isHiddenAssetGroupJob} from '../asset-graph/Utils'; + +export const extractGraphsForRepo = (repo: RepositoryGraphsFragment) => { + const jobGraphNames = new Set( + repo.pipelines.filter((p) => p.isJob && !isHiddenAssetGroupJob(p.name)).map((p) => p.graphName), + ); + + const items: Graph[] = Array.from(jobGraphNames).map((graphName) => ({ + name: graphName, + path: `/graphs/${graphName}`, + description: null, + })); + + repo.usedSolids.forEach((s) => { + if (s.definition.__typename === 'CompositeSolidDefinition') { + const invocation = s.invocations[0]; + if (invocation) { + items.push({ + name: s.definition.name, + path: `/graphs/${invocation.pipeline.name}/${invocation.solidHandle.handleID}/`, + description: s.definition.description, + }); + } + } + }); + + return items.sort((a, b) => COMMON_COLLATOR.compare(a.name, b.name)); +}; diff --git a/js_modules/dagster-ui/packages/ui-core/src/workspace/types/WorkspaceGraphsRoot.types.ts b/js_modules/dagster-ui/packages/ui-core/src/workspace/types/WorkspaceGraphsQuery.types.ts similarity index 67% rename from js_modules/dagster-ui/packages/ui-core/src/workspace/types/WorkspaceGraphsRoot.types.ts rename to js_modules/dagster-ui/packages/ui-core/src/workspace/types/WorkspaceGraphsQuery.types.ts index 08cfdade30195..0bac5463f140c 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/workspace/types/WorkspaceGraphsRoot.types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/workspace/types/WorkspaceGraphsQuery.types.ts @@ -2,6 +2,34 @@ import * as Types from '../../graphql/types'; +export type RepositoryGraphsFragment = { + __typename: 'Repository'; + id: string; + usedSolids: Array<{ + __typename: 'UsedSolid'; + definition: + | { + __typename: 'CompositeSolidDefinition'; + id: string; + name: string; + description: string | null; + } + | {__typename: 'SolidDefinition'}; + invocations: Array<{ + __typename: 'NodeInvocationSite'; + pipeline: {__typename: 'Pipeline'; id: string; name: string}; + solidHandle: {__typename: 'SolidHandle'; handleID: string}; + }>; + }>; + pipelines: Array<{ + __typename: 'Pipeline'; + id: string; + name: string; + isJob: boolean; + graphName: string; + }>; +}; + export type WorkspaceGraphsQueryVariables = Types.Exact<{ selector: Types.RepositorySelector; }>;