From cda4cfb8d83c47cfe06902fb39c4fe1866f44581 Mon Sep 17 00:00:00 2001 From: Neil Fulwiler Date: Mon, 25 Nov 2024 10:18:44 -0500 Subject: [PATCH] background asset wipe --- .../dagster-ui/packages/ui-core/client.json | 2 + .../src/assets/types/useWipeAssets.types.ts | 39 +++++++ .../ui-core/src/assets/useWipeAssets.tsx | 105 ++++++++++++++++- .../src/graphql/possibleTypes.generated.json | 2 +- .../ui-core/src/graphql/schema.graphql | 35 ++++++ .../packages/ui-core/src/graphql/types.ts | 110 ++++++++++++++++++ .../implementation/execution/__init__.py | 97 +++++++++++---- .../dagster_graphql/schema/roots/assets.py | 34 ++++++ .../dagster_graphql/schema/roots/mutation.py | 58 +++++++++ .../dagster_graphql/schema/roots/query.py | 41 ++++++- .../graphql/test_assets.py | 27 +++++ .../dagster_test/fixtures/docker_compose.py | 22 +++- .../dagster/_core/definitions/events.py | 26 +++++ .../dagster/_core/instance/__init__.py | 17 ++- .../integration_tests/test_dbt_project.py | 3 +- 15 files changed, 584 insertions(+), 34 deletions(-) diff --git a/js_modules/dagster-ui/packages/ui-core/client.json b/js_modules/dagster-ui/packages/ui-core/client.json index 9b8c430dd2ead..bddd97e59225d 100644 --- a/js_modules/dagster-ui/packages/ui-core/client.json +++ b/js_modules/dagster-ui/packages/ui-core/client.json @@ -48,6 +48,8 @@ "ReportEventPartitionDefinitionQuery": "e306421344493a9986106de14bca90ec554505d6f1965991ba502725edc41c95", "ReportEventMutation": "80b4987cdf27ec8fac25eb6b98b996bd4fdeb4cbfff605d647da5d4bb8244cb0", "AssetWipeMutation": "accefb0c47b3d4a980d16965e8af565afed787a8a987a03570df876bd734dc8f", + "BackgroundAssetWipeMutation": "855208bbb326949e82c0365a7c2d755b13dc7ddece9a6f22a90c6abb5647fae9", + "BackgroundAssetWipeStatus": "ed90ffb3b1984e03d7731ca9e12068b8706a2f0286ddb7037d9f1a271f58b85e", "RunGroupPanelQuery": "c454b4e4c3d881b2a78361c5868212f734c458291a3cb28be8ba4a63030eb004", "InstanceBackfillsQuery": "e9baee9c4eabc561ffe1ffcb06430969883c1d1cfb469438f98d821b90d3d06a", "InstanceConcurrencyLimitsQuery": "eff036379500d5b400ba5a0d3f4f22fad1bd42efefbeeafa16b43ca8b160c312", diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/types/useWipeAssets.types.ts b/js_modules/dagster-ui/packages/ui-core/src/assets/types/useWipeAssets.types.ts index 041cf552f2ebc..71004f567f329 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/types/useWipeAssets.types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/types/useWipeAssets.types.ts @@ -32,4 +32,43 @@ export type AssetWipeMutation = { | {__typename: 'UnsupportedOperationError'}; }; +export type BackgroundAssetWipeMutationVariables = Types.Exact<{ + assetPartitionRanges: Array | Types.PartitionsByAssetSelector; +}>; + +export type BackgroundAssetWipeMutation = { + __typename: 'Mutation'; + backgroundWipeAssets: + | {__typename: 'AssetNotFoundError'} + | {__typename: 'AssetWipeInProgress'; workToken: string} + | { + __typename: 'PythonError'; + message: string; + stack: Array; + errorChain: Array<{ + __typename: 'ErrorChainLink'; + isExplicitLink: boolean; + error: {__typename: 'PythonError'; message: string; stack: Array}; + }>; + } + | {__typename: 'UnauthorizedError'} + | {__typename: 'UnsupportedOperationError'}; +}; + +export type BackgroundAssetWipeStatusQueryVariables = Types.Exact<{ + workToken: Types.Scalars['String']['input']; +}>; + +export type BackgroundAssetWipeStatusQuery = { + __typename: 'Query'; + backgroundAssetWipeStatus: + | {__typename: 'BackgroundAssetWipeFailed'; failedAt: number; message: string} + | {__typename: 'BackgroundAssetWipeInProgress'; startedAt: number} + | {__typename: 'BackgroundAssetWipeSuccess'; completedAt: number}; +}; + export const AssetWipeMutationVersion = 'accefb0c47b3d4a980d16965e8af565afed787a8a987a03570df876bd734dc8f'; + +export const BackgroundAssetWipeMutationVersion = '855208bbb326949e82c0365a7c2d755b13dc7ddece9a6f22a90c6abb5647fae9'; + +export const BackgroundAssetWipeStatusVersion = 'ed90ffb3b1984e03d7731ca9e12068b8706a2f0286ddb7037d9f1a271f58b85e'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/useWipeAssets.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/useWipeAssets.tsx index 7c43657f33570..290f58b70df18 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/useWipeAssets.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/useWipeAssets.tsx @@ -1,7 +1,13 @@ import {useLayoutEffect, useRef, useState} from 'react'; -import {RefetchQueriesFunction, gql, useMutation} from '../apollo-client'; -import {AssetWipeMutation, AssetWipeMutationVariables} from './types/useWipeAssets.types'; +import {RefetchQueriesFunction, gql, useLazyQuery, useMutation} from '../apollo-client'; +import { + AssetWipeMutation, + AssetWipeMutationVariables, + BackgroundAssetWipeMutation, + BackgroundAssetWipeMutationVariables, + BackgroundAssetWipeStatusQuery, +} from './types/useWipeAssets.types'; import {showCustomAlert} from '../app/CustomAlertProvider'; import {PYTHON_ERROR_FRAGMENT} from '../app/PythonErrorFragment'; import {PartitionsByAssetSelector} from '../graphql/types'; @@ -21,6 +27,14 @@ export function useWipeAssets({ ASSET_WIPE_MUTATION, {refetchQueries}, ); + const [requestBackgroundAssetWipe] = useMutation< + BackgroundAssetWipeMutation, + BackgroundAssetWipeMutationVariables + >(BACKGROUND_ASSET_WIPE_MUTATION, {refetchQueries}); + const [ + getBackgroundWipeStatus, + {error: backgroundWipeStatusError, data: backgroundWipeStatusData}, + ] = useLazyQuery(BACKGROUND_ASSET_WIPE_STATUS); const [isWiping, setIsWiping] = useState(false); const [wipedCount, setWipedCount] = useState(0); @@ -30,6 +44,31 @@ export function useWipeAssets({ const didCancel = useRef(false); + if (isWiping && backgroundWipeStatusError) { + setFailedCount(1); + onComplete?.(); + setIsWiping(false); + } + + if (isWiping && backgroundWipeStatusData) { + const data = backgroundWipeStatusData.backgroundAssetWipeStatus; + switch (data.__typename) { + case 'BackgroundAssetWipeInProgress': + console.log('Background asset wipe in progress'); + break; + case 'BackgroundAssetWipeSuccess': + setWipedCount(1); + onComplete?.(); + setIsWiping(false); + break; + case 'BackgroundAssetWipeFailed': + setFailedCount(1); + onComplete?.(); + setIsWiping(false); + break; + } + } + const wipeAssets = async (assetPartitionRanges: PartitionsByAssetSelector[]) => { if (!assetPartitionRanges.length) { return; @@ -66,13 +105,43 @@ export function useWipeAssets({ setIsWiping(false); }; + const backgroundWipeAssets = async (assetPartitionRanges: PartitionsByAssetSelector[]) => { + if (!assetPartitionRanges.length) { + return; + } + setIsWiping(true); + const result = await requestBackgroundAssetWipe({ + variables: {assetPartitionRanges}, + refetchQueries, + }); + const data = result.data?.backgroundWipeAssets; + switch (data?.__typename) { + case 'AssetNotFoundError': + case 'PythonError': + setFailedCount(assetPartitionRanges.length); + onComplete?.(); + setIsWiping(false); + return; + case 'AssetWipeInProgress': + getBackgroundWipeStatus({variables: {workToken: data.workToken}, pollInterval: 1000}); + break; + case 'UnauthorizedError': + showCustomAlert({ + title: 'Could not wipe asset materializations', + body: 'You do not have permission to do this.', + }); + onClose(); + return; + } + }; + useLayoutEffect(() => { return () => { didCancel.current = true; }; }, []); - return {wipeAssets, isWiping, isDone, wipedCount, failedCount}; + return {backgroundWipeAssets, wipeAssets, isWiping, isDone, wipedCount, failedCount}; } export const ASSET_WIPE_MUTATION = gql` @@ -95,3 +164,33 @@ export const ASSET_WIPE_MUTATION = gql` ${PYTHON_ERROR_FRAGMENT} `; + +export const BACKGROUND_ASSET_WIPE_MUTATION = gql` + mutation BackgroundAssetWipeMutation($assetPartitionRanges: [PartitionsByAssetSelector!]!) { + backgroundWipeAssets(assetPartitionRanges: $assetPartitionRanges) { + ... on AssetWipeInProgress { + workToken + } + ...PythonErrorFragment + } + } + + ${PYTHON_ERROR_FRAGMENT} +`; + +export const BACKGROUND_ASSET_WIPE_STATUS = gql` + query BackgroundAssetWipeStatus($workToken: String!) { + backgroundAssetWipeStatus(workToken: $workToken) { + ... on BackgroundAssetWipeSuccess { + completedAt + } + ... on BackgroundAssetWipeInProgress { + startedAt + } + ... on BackgroundAssetWipeFailed { + failedAt + message + } + } + } +`; diff --git a/js_modules/dagster-ui/packages/ui-core/src/graphql/possibleTypes.generated.json b/js_modules/dagster-ui/packages/ui-core/src/graphql/possibleTypes.generated.json index 25046b8064e0f..526322604c456 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/graphql/possibleTypes.generated.json +++ b/js_modules/dagster-ui/packages/ui-core/src/graphql/possibleTypes.generated.json @@ -1 +1 @@ -{"DisplayableEvent":["EngineEvent","ExecutionStepOutputEvent","ExpectationResult","FailureMetadata","HandledOutputEvent","LoadedInputEvent","ObjectStoreOperationResult","ResourceInitFailureEvent","ResourceInitStartedEvent","ResourceInitSuccessEvent","StepWorkerStartedEvent","StepWorkerStartingEvent","MaterializationEvent","ObservationEvent","TypeCheck"],"MarkerEvent":["EngineEvent","ResourceInitFailureEvent","ResourceInitStartedEvent","ResourceInitSuccessEvent","StepWorkerStartedEvent","StepWorkerStartingEvent"],"ErrorEvent":["EngineEvent","ExecutionStepFailureEvent","ExecutionStepUpForRetryEvent","HookErroredEvent","RunCanceledEvent","RunFailureEvent","ResourceInitFailureEvent"],"MessageEvent":["EngineEvent","ExecutionStepFailureEvent","ExecutionStepInputEvent","ExecutionStepOutputEvent","ExecutionStepRestartEvent","ExecutionStepSkippedEvent","ExecutionStepStartEvent","ExecutionStepSuccessEvent","ExecutionStepUpForRetryEvent","HandledOutputEvent","HookCompletedEvent","HookErroredEvent","HookSkippedEvent","LoadedInputEvent","LogMessageEvent","ObjectStoreOperationEvent","RunCanceledEvent","RunCancelingEvent","RunDequeuedEvent","RunEnqueuedEvent","RunFailureEvent","ResourceInitFailureEvent","ResourceInitStartedEvent","ResourceInitSuccessEvent","RunStartEvent","RunStartingEvent","RunSuccessEvent","StepExpectationResultEvent","StepWorkerStartedEvent","StepWorkerStartingEvent","MaterializationEvent","ObservationEvent","AssetMaterializationPlannedEvent","LogsCapturedEvent","AlertStartEvent","AlertSuccessEvent","AlertFailureEvent","AssetCheckEvaluationPlannedEvent","AssetCheckEvaluationEvent"],"RunEvent":["RunCanceledEvent","RunCancelingEvent","RunDequeuedEvent","RunEnqueuedEvent","RunFailureEvent","RunStartEvent","RunStartingEvent","RunSuccessEvent","AssetMaterializationPlannedEvent","AlertStartEvent","AlertSuccessEvent","AlertFailureEvent"],"PipelineRunStepStats":["RunStepStats"],"StepEvent":["EngineEvent","ExecutionStepFailureEvent","ExecutionStepInputEvent","ExecutionStepOutputEvent","ExecutionStepRestartEvent","ExecutionStepSkippedEvent","ExecutionStepStartEvent","ExecutionStepSuccessEvent","ExecutionStepUpForRetryEvent","HandledOutputEvent","HookCompletedEvent","HookErroredEvent","HookSkippedEvent","LoadedInputEvent","ObjectStoreOperationEvent","ResourceInitFailureEvent","ResourceInitStartedEvent","ResourceInitSuccessEvent","StepExpectationResultEvent","StepWorkerStartedEvent","StepWorkerStartingEvent","MaterializationEvent","ObservationEvent","AssetCheckEvaluationPlannedEvent","AssetCheckEvaluationEvent"],"AssetOwner":["UserAssetOwner","TeamAssetOwner"],"AssetPartitionStatuses":["DefaultPartitionStatuses","MultiPartitionStatuses","TimePartitionStatuses"],"PartitionStatus1D":["TimePartitionStatuses","DefaultPartitionStatuses"],"AssetChecksOrError":["AssetChecks","AssetCheckNeedsMigrationError","AssetCheckNeedsUserCodeUpgrade","AssetCheckNeedsAgentUpgradeError"],"Instigator":["Schedule","Sensor"],"EvaluationStackEntry":["EvaluationStackListItemEntry","EvaluationStackPathEntry","EvaluationStackMapKeyEntry","EvaluationStackMapValueEntry"],"IPipelineSnapshot":["Pipeline","PipelineSnapshot","Job"],"PipelineConfigValidationError":["FieldNotDefinedConfigError","FieldsNotDefinedConfigError","MissingFieldConfigError","MissingFieldsConfigError","RuntimeMismatchConfigError","SelectorTypeConfigError"],"PipelineConfigValidationInvalid":["RunConfigValidationInvalid"],"PipelineConfigValidationResult":["InvalidSubsetError","PipelineConfigValidationValid","RunConfigValidationInvalid","PipelineNotFoundError","PythonError"],"PipelineReference":["PipelineSnapshot","UnknownPipeline"],"PipelineRun":["Run"],"DagsterRunEvent":["ExecutionStepFailureEvent","ExecutionStepInputEvent","ExecutionStepOutputEvent","ExecutionStepSkippedEvent","ExecutionStepStartEvent","ExecutionStepSuccessEvent","ExecutionStepUpForRetryEvent","ExecutionStepRestartEvent","LogMessageEvent","ResourceInitFailureEvent","ResourceInitStartedEvent","ResourceInitSuccessEvent","RunFailureEvent","RunStartEvent","RunEnqueuedEvent","RunDequeuedEvent","RunStartingEvent","RunCancelingEvent","RunCanceledEvent","RunSuccessEvent","StepWorkerStartedEvent","StepWorkerStartingEvent","HandledOutputEvent","LoadedInputEvent","LogsCapturedEvent","ObjectStoreOperationEvent","StepExpectationResultEvent","MaterializationEvent","ObservationEvent","EngineEvent","HookCompletedEvent","HookSkippedEvent","HookErroredEvent","AlertStartEvent","AlertSuccessEvent","AlertFailureEvent","AssetMaterializationPlannedEvent","AssetCheckEvaluationPlannedEvent","AssetCheckEvaluationEvent"],"PipelineRunLogsSubscriptionPayload":["PipelineRunLogsSubscriptionSuccess","PipelineRunLogsSubscriptionFailure"],"RunOrError":["Run","RunNotFoundError","PythonError"],"PipelineRunStatsSnapshot":["RunStatsSnapshot"],"RunStatsSnapshotOrError":["RunStatsSnapshot","PythonError"],"PipelineSnapshotOrError":["PipelineNotFoundError","PipelineSnapshot","PipelineSnapshotNotFoundError","PythonError"],"RunsFeedEntry":["Run","PartitionBackfill"],"AssetOrError":["Asset","AssetNotFoundError"],"AssetsOrError":["AssetConnection","PythonError"],"DeletePipelineRunResult":["DeletePipelineRunSuccess","UnauthorizedError","PythonError","RunNotFoundError"],"ExecutionPlanOrError":["ExecutionPlan","RunConfigValidationInvalid","PipelineNotFoundError","InvalidSubsetError","PythonError"],"LaunchMultipleRunsResultOrError":["LaunchMultipleRunsResult","PythonError"],"PipelineOrError":["Pipeline","PipelineNotFoundError","InvalidSubsetError","PythonError"],"ReloadRepositoryLocationMutationResult":["WorkspaceLocationEntry","ReloadNotSupported","RepositoryLocationNotFound","UnauthorizedError","PythonError"],"RepositoryLocationOrLoadError":["RepositoryLocation","PythonError"],"ReloadWorkspaceMutationResult":["Workspace","UnauthorizedError","PythonError"],"ShutdownRepositoryLocationMutationResult":["ShutdownRepositoryLocationSuccess","RepositoryLocationNotFound","UnauthorizedError","PythonError"],"TerminatePipelineExecutionFailure":["TerminateRunFailure"],"TerminatePipelineExecutionSuccess":["TerminateRunSuccess"],"TerminateRunResult":["TerminateRunSuccess","TerminateRunFailure","RunNotFoundError","UnauthorizedError","PythonError"],"ScheduleMutationResult":["PythonError","UnauthorizedError","ScheduleStateResult","ScheduleNotFoundError"],"ScheduleOrError":["Schedule","ScheduleNotFoundError","PythonError"],"SchedulerOrError":["Scheduler","SchedulerNotDefinedError","PythonError"],"SchedulesOrError":["Schedules","RepositoryNotFoundError","PythonError"],"ScheduleTickSpecificData":["ScheduleTickSuccessData","ScheduleTickFailureData"],"LaunchBackfillResult":["LaunchBackfillSuccess","PartitionSetNotFoundError","PartitionKeysNotFoundError","InvalidStepError","InvalidOutputError","RunConfigValidationInvalid","PipelineNotFoundError","RunConflict","UnauthorizedError","PythonError","InvalidSubsetError","PresetNotFoundError","ConflictingExecutionParamsError","NoModeProvidedError"],"ConfigTypeOrError":["EnumConfigType","CompositeConfigType","RegularConfigType","PipelineNotFoundError","ConfigTypeNotFoundError","PythonError"],"ConfigType":["ArrayConfigType","CompositeConfigType","EnumConfigType","NullableConfigType","RegularConfigType","ScalarUnionConfigType","MapConfigType"],"WrappingConfigType":["ArrayConfigType","NullableConfigType"],"DagsterType":["ListDagsterType","NullableDagsterType","RegularDagsterType"],"DagsterTypeOrError":["RegularDagsterType","PipelineNotFoundError","DagsterTypeNotFoundError","PythonError"],"WrappingDagsterType":["ListDagsterType","NullableDagsterType"],"Error":["AssetCheckNeedsMigrationError","AssetCheckNeedsUserCodeUpgrade","AssetCheckNeedsAgentUpgradeError","PartitionKeysNotFoundError","AssetNotFoundError","ConflictingExecutionParamsError","ConfigTypeNotFoundError","DagsterTypeNotFoundError","InvalidPipelineRunsFilterError","InvalidSubsetError","ModeNotFoundError","NoModeProvidedError","PartitionSetNotFoundError","PipelineNotFoundError","RunConflict","PipelineSnapshotNotFoundError","PresetNotFoundError","PythonError","ErrorChainLink","UnauthorizedError","ReloadNotSupported","RepositoryLocationNotFound","RepositoryNotFoundError","ResourceNotFoundError","RunGroupNotFoundError","RunNotFoundError","ScheduleNotFoundError","SchedulerNotDefinedError","SensorNotFoundError","UnsupportedOperationError","DuplicateDynamicPartitionError","InstigationStateNotFoundError","SolidStepStatusUnavailableError","GraphNotFoundError","BackfillNotFoundError","PartitionSubsetDeserializationError","AutoMaterializeAssetEvaluationNeedsMigrationError"],"PipelineRunConflict":["RunConflict"],"PipelineRunNotFoundError":["RunNotFoundError"],"RepositoriesOrError":["RepositoryConnection","RepositoryNotFoundError","PythonError"],"RepositoryOrError":["PythonError","Repository","RepositoryNotFoundError"],"WorkspaceLocationEntryOrError":["WorkspaceLocationEntry","PythonError"],"InstigationTypeSpecificData":["SensorData","ScheduleData"],"InstigationStateOrError":["InstigationState","InstigationStateNotFoundError","PythonError"],"InstigationStatesOrError":["InstigationStates","PythonError"],"MetadataEntry":["TableColumnLineageMetadataEntry","TableSchemaMetadataEntry","TableMetadataEntry","FloatMetadataEntry","IntMetadataEntry","JsonMetadataEntry","BoolMetadataEntry","MarkdownMetadataEntry","PathMetadataEntry","NotebookMetadataEntry","PythonArtifactMetadataEntry","TextMetadataEntry","UrlMetadataEntry","PipelineRunMetadataEntry","AssetMetadataEntry","JobMetadataEntry","CodeReferencesMetadataEntry","NullMetadataEntry","TimestampMetadataEntry"],"SourceLocation":["LocalFileCodeReference","UrlCodeReference"],"PartitionRunConfigOrError":["PartitionRunConfig","PythonError"],"AssetBackfillStatus":["AssetPartitionsStatusCounts","UnpartitionedAssetStatus"],"PartitionSetOrError":["PartitionSet","PartitionSetNotFoundError","PythonError"],"PartitionSetsOrError":["PartitionSets","PipelineNotFoundError","PythonError"],"PartitionsOrError":["Partitions","PythonError"],"PartitionStatusesOrError":["PartitionStatuses","PythonError"],"PartitionTagsOrError":["PartitionTags","PythonError"],"RunConfigSchemaOrError":["RunConfigSchema","PipelineNotFoundError","InvalidSubsetError","ModeNotFoundError","PythonError"],"LaunchRunResult":["LaunchRunSuccess","InvalidStepError","InvalidOutputError","RunConfigValidationInvalid","PipelineNotFoundError","RunConflict","UnauthorizedError","PythonError","InvalidSubsetError","PresetNotFoundError","ConflictingExecutionParamsError","NoModeProvidedError"],"LaunchRunReexecutionResult":["LaunchRunSuccess","InvalidStepError","InvalidOutputError","RunConfigValidationInvalid","PipelineNotFoundError","RunConflict","UnauthorizedError","PythonError","InvalidSubsetError","PresetNotFoundError","ConflictingExecutionParamsError","NoModeProvidedError"],"LaunchPipelineRunSuccess":["LaunchRunSuccess"],"RunsOrError":["Runs","InvalidPipelineRunsFilterError","PythonError"],"PipelineRuns":["Runs"],"RunGroupOrError":["RunGroup","RunGroupNotFoundError","PythonError"],"SensorOrError":["Sensor","SensorNotFoundError","UnauthorizedError","PythonError"],"SensorsOrError":["Sensors","RepositoryNotFoundError","PythonError"],"StopSensorMutationResultOrError":["StopSensorMutationResult","UnauthorizedError","PythonError"],"ISolidDefinition":["CompositeSolidDefinition","SolidDefinition"],"SolidContainer":["Pipeline","PipelineSnapshot","Job","CompositeSolidDefinition","Graph"],"SolidStepStatsOrError":["SolidStepStatsConnection","SolidStepStatusUnavailableError"],"WorkspaceOrError":["Workspace","PythonError"],"WorkspaceLocationStatusEntriesOrError":["WorkspaceLocationStatusEntries","PythonError"],"ResourcesOrError":["ResourceConnection","PipelineNotFoundError","InvalidSubsetError","PythonError"],"GraphOrError":["Graph","GraphNotFoundError","PythonError"],"ResourceDetailsOrError":["ResourceDetails","ResourceNotFoundError","PythonError"],"ResourceDetailsListOrError":["ResourceDetailsList","RepositoryNotFoundError","PythonError"],"EnvVarWithConsumersOrError":["EnvVarWithConsumersList","PythonError"],"RunsFeedConnectionOrError":["RunsFeedConnection","PythonError"],"RunsFeedCountOrError":["RunsFeedCount","PythonError"],"RunTagKeysOrError":["PythonError","RunTagKeys"],"RunTagsOrError":["PythonError","RunTags"],"RunIdsOrError":["RunIds","InvalidPipelineRunsFilterError","PythonError"],"AssetNodeOrError":["AssetNode","AssetNotFoundError"],"PartitionBackfillOrError":["PartitionBackfill","BackfillNotFoundError","PythonError"],"PartitionBackfillsOrError":["PartitionBackfills","PythonError"],"EventConnectionOrError":["EventConnection","RunNotFoundError","PythonError"],"AutoMaterializeAssetEvaluationRecordsOrError":["AutoMaterializeAssetEvaluationRecords","AutoMaterializeAssetEvaluationNeedsMigrationError"],"PartitionKeysOrError":["PartitionKeys","PartitionSubsetDeserializationError"],"AutoMaterializeRuleEvaluationData":["TextRuleEvaluationData","ParentMaterializedRuleEvaluationData","WaitingOnKeysRuleEvaluationData"],"AssetConditionEvaluationNode":["UnpartitionedAssetConditionEvaluationNode","PartitionedAssetConditionEvaluationNode","SpecificPartitionAssetConditionEvaluationNode"],"AssetConditionEvaluationRecordsOrError":["AssetConditionEvaluationRecords","AutoMaterializeAssetEvaluationNeedsMigrationError"],"SensorDryRunResult":["PythonError","SensorNotFoundError","DryRunInstigationTick"],"ScheduleDryRunResult":["DryRunInstigationTick","PythonError","ScheduleNotFoundError"],"TerminateRunsResultOrError":["TerminateRunsResult","PythonError"],"AssetWipeMutationResult":["AssetNotFoundError","UnauthorizedError","PythonError","UnsupportedOperationError","AssetWipeSuccess"],"ReportRunlessAssetEventsResult":["UnauthorizedError","PythonError","ReportRunlessAssetEventsSuccess"],"ResumeBackfillResult":["ResumeBackfillSuccess","UnauthorizedError","PythonError"],"CancelBackfillResult":["CancelBackfillSuccess","UnauthorizedError","PythonError"],"LogTelemetryMutationResult":["LogTelemetrySuccess","PythonError"],"AddDynamicPartitionResult":["AddDynamicPartitionSuccess","UnauthorizedError","PythonError","DuplicateDynamicPartitionError"],"DeleteDynamicPartitionsResult":["DeleteDynamicPartitionsSuccess","UnauthorizedError","PythonError"]} \ No newline at end of file +{"DisplayableEvent":["EngineEvent","ExecutionStepOutputEvent","ExpectationResult","FailureMetadata","HandledOutputEvent","LoadedInputEvent","ObjectStoreOperationResult","ResourceInitFailureEvent","ResourceInitStartedEvent","ResourceInitSuccessEvent","StepWorkerStartedEvent","StepWorkerStartingEvent","MaterializationEvent","ObservationEvent","TypeCheck"],"MarkerEvent":["EngineEvent","ResourceInitFailureEvent","ResourceInitStartedEvent","ResourceInitSuccessEvent","StepWorkerStartedEvent","StepWorkerStartingEvent"],"ErrorEvent":["EngineEvent","ExecutionStepFailureEvent","ExecutionStepUpForRetryEvent","HookErroredEvent","RunCanceledEvent","RunFailureEvent","ResourceInitFailureEvent"],"MessageEvent":["EngineEvent","ExecutionStepFailureEvent","ExecutionStepInputEvent","ExecutionStepOutputEvent","ExecutionStepRestartEvent","ExecutionStepSkippedEvent","ExecutionStepStartEvent","ExecutionStepSuccessEvent","ExecutionStepUpForRetryEvent","HandledOutputEvent","HookCompletedEvent","HookErroredEvent","HookSkippedEvent","LoadedInputEvent","LogMessageEvent","ObjectStoreOperationEvent","RunCanceledEvent","RunCancelingEvent","RunDequeuedEvent","RunEnqueuedEvent","RunFailureEvent","ResourceInitFailureEvent","ResourceInitStartedEvent","ResourceInitSuccessEvent","RunStartEvent","RunStartingEvent","RunSuccessEvent","StepExpectationResultEvent","StepWorkerStartedEvent","StepWorkerStartingEvent","MaterializationEvent","ObservationEvent","AssetMaterializationPlannedEvent","LogsCapturedEvent","AlertStartEvent","AlertSuccessEvent","AlertFailureEvent","AssetCheckEvaluationPlannedEvent","AssetCheckEvaluationEvent"],"RunEvent":["RunCanceledEvent","RunCancelingEvent","RunDequeuedEvent","RunEnqueuedEvent","RunFailureEvent","RunStartEvent","RunStartingEvent","RunSuccessEvent","AssetMaterializationPlannedEvent","AlertStartEvent","AlertSuccessEvent","AlertFailureEvent"],"PipelineRunStepStats":["RunStepStats"],"StepEvent":["EngineEvent","ExecutionStepFailureEvent","ExecutionStepInputEvent","ExecutionStepOutputEvent","ExecutionStepRestartEvent","ExecutionStepSkippedEvent","ExecutionStepStartEvent","ExecutionStepSuccessEvent","ExecutionStepUpForRetryEvent","HandledOutputEvent","HookCompletedEvent","HookErroredEvent","HookSkippedEvent","LoadedInputEvent","ObjectStoreOperationEvent","ResourceInitFailureEvent","ResourceInitStartedEvent","ResourceInitSuccessEvent","StepExpectationResultEvent","StepWorkerStartedEvent","StepWorkerStartingEvent","MaterializationEvent","ObservationEvent","AssetCheckEvaluationPlannedEvent","AssetCheckEvaluationEvent"],"AssetOwner":["UserAssetOwner","TeamAssetOwner"],"AssetPartitionStatuses":["DefaultPartitionStatuses","MultiPartitionStatuses","TimePartitionStatuses"],"PartitionStatus1D":["TimePartitionStatuses","DefaultPartitionStatuses"],"AssetChecksOrError":["AssetChecks","AssetCheckNeedsMigrationError","AssetCheckNeedsUserCodeUpgrade","AssetCheckNeedsAgentUpgradeError"],"Instigator":["Schedule","Sensor"],"EvaluationStackEntry":["EvaluationStackListItemEntry","EvaluationStackPathEntry","EvaluationStackMapKeyEntry","EvaluationStackMapValueEntry"],"IPipelineSnapshot":["Pipeline","PipelineSnapshot","Job"],"PipelineConfigValidationError":["FieldNotDefinedConfigError","FieldsNotDefinedConfigError","MissingFieldConfigError","MissingFieldsConfigError","RuntimeMismatchConfigError","SelectorTypeConfigError"],"PipelineConfigValidationInvalid":["RunConfigValidationInvalid"],"PipelineConfigValidationResult":["InvalidSubsetError","PipelineConfigValidationValid","RunConfigValidationInvalid","PipelineNotFoundError","PythonError"],"PipelineReference":["PipelineSnapshot","UnknownPipeline"],"PipelineRun":["Run"],"DagsterRunEvent":["ExecutionStepFailureEvent","ExecutionStepInputEvent","ExecutionStepOutputEvent","ExecutionStepSkippedEvent","ExecutionStepStartEvent","ExecutionStepSuccessEvent","ExecutionStepUpForRetryEvent","ExecutionStepRestartEvent","LogMessageEvent","ResourceInitFailureEvent","ResourceInitStartedEvent","ResourceInitSuccessEvent","RunFailureEvent","RunStartEvent","RunEnqueuedEvent","RunDequeuedEvent","RunStartingEvent","RunCancelingEvent","RunCanceledEvent","RunSuccessEvent","StepWorkerStartedEvent","StepWorkerStartingEvent","HandledOutputEvent","LoadedInputEvent","LogsCapturedEvent","ObjectStoreOperationEvent","StepExpectationResultEvent","MaterializationEvent","ObservationEvent","EngineEvent","HookCompletedEvent","HookSkippedEvent","HookErroredEvent","AlertStartEvent","AlertSuccessEvent","AlertFailureEvent","AssetMaterializationPlannedEvent","AssetCheckEvaluationPlannedEvent","AssetCheckEvaluationEvent"],"PipelineRunLogsSubscriptionPayload":["PipelineRunLogsSubscriptionSuccess","PipelineRunLogsSubscriptionFailure"],"RunOrError":["Run","RunNotFoundError","PythonError"],"PipelineRunStatsSnapshot":["RunStatsSnapshot"],"RunStatsSnapshotOrError":["RunStatsSnapshot","PythonError"],"PipelineSnapshotOrError":["PipelineNotFoundError","PipelineSnapshot","PipelineSnapshotNotFoundError","PythonError"],"RunsFeedEntry":["Run","PartitionBackfill"],"AssetOrError":["Asset","AssetNotFoundError"],"AssetsOrError":["AssetConnection","PythonError"],"DeletePipelineRunResult":["DeletePipelineRunSuccess","UnauthorizedError","PythonError","RunNotFoundError"],"ExecutionPlanOrError":["ExecutionPlan","RunConfigValidationInvalid","PipelineNotFoundError","InvalidSubsetError","PythonError"],"LaunchMultipleRunsResultOrError":["LaunchMultipleRunsResult","PythonError"],"PipelineOrError":["Pipeline","PipelineNotFoundError","InvalidSubsetError","PythonError"],"ReloadRepositoryLocationMutationResult":["WorkspaceLocationEntry","ReloadNotSupported","RepositoryLocationNotFound","UnauthorizedError","PythonError"],"RepositoryLocationOrLoadError":["RepositoryLocation","PythonError"],"ReloadWorkspaceMutationResult":["Workspace","UnauthorizedError","PythonError"],"ShutdownRepositoryLocationMutationResult":["ShutdownRepositoryLocationSuccess","RepositoryLocationNotFound","UnauthorizedError","PythonError"],"TerminatePipelineExecutionFailure":["TerminateRunFailure"],"TerminatePipelineExecutionSuccess":["TerminateRunSuccess"],"TerminateRunResult":["TerminateRunSuccess","TerminateRunFailure","RunNotFoundError","UnauthorizedError","PythonError"],"ScheduleMutationResult":["PythonError","UnauthorizedError","ScheduleStateResult","ScheduleNotFoundError"],"ScheduleOrError":["Schedule","ScheduleNotFoundError","PythonError"],"SchedulerOrError":["Scheduler","SchedulerNotDefinedError","PythonError"],"SchedulesOrError":["Schedules","RepositoryNotFoundError","PythonError"],"ScheduleTickSpecificData":["ScheduleTickSuccessData","ScheduleTickFailureData"],"LaunchBackfillResult":["LaunchBackfillSuccess","PartitionSetNotFoundError","PartitionKeysNotFoundError","InvalidStepError","InvalidOutputError","RunConfigValidationInvalid","PipelineNotFoundError","RunConflict","UnauthorizedError","PythonError","InvalidSubsetError","PresetNotFoundError","ConflictingExecutionParamsError","NoModeProvidedError"],"ConfigTypeOrError":["EnumConfigType","CompositeConfigType","RegularConfigType","PipelineNotFoundError","ConfigTypeNotFoundError","PythonError"],"ConfigType":["ArrayConfigType","CompositeConfigType","EnumConfigType","NullableConfigType","RegularConfigType","ScalarUnionConfigType","MapConfigType"],"WrappingConfigType":["ArrayConfigType","NullableConfigType"],"DagsterType":["ListDagsterType","NullableDagsterType","RegularDagsterType"],"DagsterTypeOrError":["RegularDagsterType","PipelineNotFoundError","DagsterTypeNotFoundError","PythonError"],"WrappingDagsterType":["ListDagsterType","NullableDagsterType"],"Error":["AssetCheckNeedsMigrationError","AssetCheckNeedsUserCodeUpgrade","AssetCheckNeedsAgentUpgradeError","PartitionKeysNotFoundError","AssetNotFoundError","ConflictingExecutionParamsError","ConfigTypeNotFoundError","DagsterTypeNotFoundError","InvalidPipelineRunsFilterError","InvalidSubsetError","ModeNotFoundError","NoModeProvidedError","PartitionSetNotFoundError","PipelineNotFoundError","RunConflict","PipelineSnapshotNotFoundError","PresetNotFoundError","PythonError","ErrorChainLink","UnauthorizedError","ReloadNotSupported","RepositoryLocationNotFound","RepositoryNotFoundError","ResourceNotFoundError","RunGroupNotFoundError","RunNotFoundError","ScheduleNotFoundError","SchedulerNotDefinedError","SensorNotFoundError","UnsupportedOperationError","DuplicateDynamicPartitionError","InstigationStateNotFoundError","SolidStepStatusUnavailableError","GraphNotFoundError","BackfillNotFoundError","PartitionSubsetDeserializationError","AutoMaterializeAssetEvaluationNeedsMigrationError"],"PipelineRunConflict":["RunConflict"],"PipelineRunNotFoundError":["RunNotFoundError"],"RepositoriesOrError":["RepositoryConnection","RepositoryNotFoundError","PythonError"],"RepositoryOrError":["PythonError","Repository","RepositoryNotFoundError"],"WorkspaceLocationEntryOrError":["WorkspaceLocationEntry","PythonError"],"InstigationTypeSpecificData":["SensorData","ScheduleData"],"InstigationStateOrError":["InstigationState","InstigationStateNotFoundError","PythonError"],"InstigationStatesOrError":["InstigationStates","PythonError"],"MetadataEntry":["TableColumnLineageMetadataEntry","TableSchemaMetadataEntry","TableMetadataEntry","FloatMetadataEntry","IntMetadataEntry","JsonMetadataEntry","BoolMetadataEntry","MarkdownMetadataEntry","PathMetadataEntry","NotebookMetadataEntry","PythonArtifactMetadataEntry","TextMetadataEntry","UrlMetadataEntry","PipelineRunMetadataEntry","AssetMetadataEntry","JobMetadataEntry","CodeReferencesMetadataEntry","NullMetadataEntry","TimestampMetadataEntry"],"SourceLocation":["LocalFileCodeReference","UrlCodeReference"],"PartitionRunConfigOrError":["PartitionRunConfig","PythonError"],"AssetBackfillStatus":["AssetPartitionsStatusCounts","UnpartitionedAssetStatus"],"PartitionSetOrError":["PartitionSet","PartitionSetNotFoundError","PythonError"],"PartitionSetsOrError":["PartitionSets","PipelineNotFoundError","PythonError"],"PartitionsOrError":["Partitions","PythonError"],"PartitionStatusesOrError":["PartitionStatuses","PythonError"],"PartitionTagsOrError":["PartitionTags","PythonError"],"RunConfigSchemaOrError":["RunConfigSchema","PipelineNotFoundError","InvalidSubsetError","ModeNotFoundError","PythonError"],"LaunchRunResult":["LaunchRunSuccess","InvalidStepError","InvalidOutputError","RunConfigValidationInvalid","PipelineNotFoundError","RunConflict","UnauthorizedError","PythonError","InvalidSubsetError","PresetNotFoundError","ConflictingExecutionParamsError","NoModeProvidedError"],"LaunchRunReexecutionResult":["LaunchRunSuccess","InvalidStepError","InvalidOutputError","RunConfigValidationInvalid","PipelineNotFoundError","RunConflict","UnauthorizedError","PythonError","InvalidSubsetError","PresetNotFoundError","ConflictingExecutionParamsError","NoModeProvidedError"],"LaunchPipelineRunSuccess":["LaunchRunSuccess"],"RunsOrError":["Runs","InvalidPipelineRunsFilterError","PythonError"],"PipelineRuns":["Runs"],"RunGroupOrError":["RunGroup","RunGroupNotFoundError","PythonError"],"SensorOrError":["Sensor","SensorNotFoundError","UnauthorizedError","PythonError"],"SensorsOrError":["Sensors","RepositoryNotFoundError","PythonError"],"StopSensorMutationResultOrError":["StopSensorMutationResult","UnauthorizedError","PythonError"],"ISolidDefinition":["CompositeSolidDefinition","SolidDefinition"],"SolidContainer":["Pipeline","PipelineSnapshot","Job","CompositeSolidDefinition","Graph"],"SolidStepStatsOrError":["SolidStepStatsConnection","SolidStepStatusUnavailableError"],"WorkspaceOrError":["Workspace","PythonError"],"WorkspaceLocationStatusEntriesOrError":["WorkspaceLocationStatusEntries","PythonError"],"ResourcesOrError":["ResourceConnection","PipelineNotFoundError","InvalidSubsetError","PythonError"],"GraphOrError":["Graph","GraphNotFoundError","PythonError"],"ResourceDetailsOrError":["ResourceDetails","ResourceNotFoundError","PythonError"],"ResourceDetailsListOrError":["ResourceDetailsList","RepositoryNotFoundError","PythonError"],"EnvVarWithConsumersOrError":["EnvVarWithConsumersList","PythonError"],"RunsFeedConnectionOrError":["RunsFeedConnection","PythonError"],"RunsFeedCountOrError":["RunsFeedCount","PythonError"],"RunTagKeysOrError":["PythonError","RunTagKeys"],"RunTagsOrError":["PythonError","RunTags"],"RunIdsOrError":["RunIds","InvalidPipelineRunsFilterError","PythonError"],"AssetNodeOrError":["AssetNode","AssetNotFoundError"],"PartitionBackfillOrError":["PartitionBackfill","BackfillNotFoundError","PythonError"],"BackgroundAssetWipeStatus":["BackgroundAssetWipeSuccess","BackgroundAssetWipeInProgress","BackgroundAssetWipeFailed"],"PartitionBackfillsOrError":["PartitionBackfills","PythonError"],"EventConnectionOrError":["EventConnection","RunNotFoundError","PythonError"],"AutoMaterializeAssetEvaluationRecordsOrError":["AutoMaterializeAssetEvaluationRecords","AutoMaterializeAssetEvaluationNeedsMigrationError"],"PartitionKeysOrError":["PartitionKeys","PartitionSubsetDeserializationError"],"AutoMaterializeRuleEvaluationData":["TextRuleEvaluationData","ParentMaterializedRuleEvaluationData","WaitingOnKeysRuleEvaluationData"],"AssetConditionEvaluationNode":["UnpartitionedAssetConditionEvaluationNode","PartitionedAssetConditionEvaluationNode","SpecificPartitionAssetConditionEvaluationNode"],"AssetConditionEvaluationRecordsOrError":["AssetConditionEvaluationRecords","AutoMaterializeAssetEvaluationNeedsMigrationError"],"SensorDryRunResult":["PythonError","SensorNotFoundError","DryRunInstigationTick"],"ScheduleDryRunResult":["DryRunInstigationTick","PythonError","ScheduleNotFoundError"],"TerminateRunsResultOrError":["TerminateRunsResult","PythonError"],"AssetWipeMutationResult":["AssetNotFoundError","UnauthorizedError","PythonError","UnsupportedOperationError","AssetWipeSuccess"],"BackgroundAssetWipeMutationResult":["AssetNotFoundError","UnauthorizedError","PythonError","UnsupportedOperationError","AssetWipeInProgress"],"ReportRunlessAssetEventsResult":["UnauthorizedError","PythonError","ReportRunlessAssetEventsSuccess"],"ResumeBackfillResult":["ResumeBackfillSuccess","UnauthorizedError","PythonError"],"CancelBackfillResult":["CancelBackfillSuccess","UnauthorizedError","PythonError"],"LogTelemetryMutationResult":["LogTelemetrySuccess","PythonError"],"AddDynamicPartitionResult":["AddDynamicPartitionSuccess","UnauthorizedError","PythonError","DuplicateDynamicPartitionError"],"DeleteDynamicPartitionsResult":["DeleteDynamicPartitionsSuccess","UnauthorizedError","PythonError"]} \ No newline at end of file diff --git a/js_modules/dagster-ui/packages/ui-core/src/graphql/schema.graphql b/js_modules/dagster-ui/packages/ui-core/src/graphql/schema.graphql index 2f4c3193df0c9..41d38b2b03b12 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/graphql/schema.graphql +++ b/js_modules/dagster-ui/packages/ui-core/src/graphql/schema.graphql @@ -3345,6 +3345,7 @@ type Query { assetNodeDefinitionCollisions(assetKeys: [AssetKeyInput!]!): [AssetNodeDefinitionCollision!]! partitionBackfillOrError(backfillId: String!): PartitionBackfillOrError! assetBackfillPreview(params: AssetBackfillPreviewParams!): [AssetPartitions!]! + backgroundAssetWipeStatus(workToken: String!): BackgroundAssetWipeStatus! partitionBackfillsOrError( status: BulkActionStatus cursor: String @@ -3544,6 +3545,26 @@ input AssetBackfillPreviewParams { assetSelection: [AssetKeyInput!]! } +union BackgroundAssetWipeStatus = + | BackgroundAssetWipeSuccess + | BackgroundAssetWipeInProgress + | BackgroundAssetWipeFailed + +type BackgroundAssetWipeSuccess { + startedAt: Float! + completedAt: Float! +} + +type BackgroundAssetWipeInProgress { + startedAt: Float! +} + +type BackgroundAssetWipeFailed { + startedAt: Float! + failedAt: Float! + message: String! +} + union PartitionBackfillsOrError = PartitionBackfills | PythonError type PartitionBackfills { @@ -3757,6 +3778,9 @@ type Mutation { repositoryLocationName: String! ): ShutdownRepositoryLocationMutationResult! wipeAssets(assetPartitionRanges: [PartitionsByAssetSelector!]!): AssetWipeMutationResult! + backgroundWipeAssets( + assetPartitionRanges: [PartitionsByAssetSelector!]! + ): BackgroundAssetWipeMutationResult! reportRunlessAssetEvents( eventParams: ReportRunlessAssetEventsParams! ): ReportRunlessAssetEventsResult! @@ -3829,6 +3853,17 @@ type PartitionRange { end: String! } +union BackgroundAssetWipeMutationResult = + | AssetNotFoundError + | UnauthorizedError + | PythonError + | UnsupportedOperationError + | AssetWipeInProgress + +type AssetWipeInProgress { + workToken: String! +} + union ReportRunlessAssetEventsResult = | UnauthorizedError | PythonError diff --git a/js_modules/dagster-ui/packages/ui-core/src/graphql/types.ts b/js_modules/dagster-ui/packages/ui-core/src/graphql/types.ts index ed57441d95d09..1a10acfd73006 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/graphql/types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/graphql/types.ts @@ -601,6 +601,11 @@ export type AssetSelection = { assetsOrError: AssetsOrError; }; +export type AssetWipeInProgress = { + __typename: 'AssetWipeInProgress'; + workToken: Scalars['String']['output']; +}; + export type AssetWipeMutationResult = | AssetNotFoundError | AssetWipeSuccess @@ -722,6 +727,36 @@ export enum BackfillPolicyType { SINGLE_RUN = 'SINGLE_RUN', } +export type BackgroundAssetWipeFailed = { + __typename: 'BackgroundAssetWipeFailed'; + failedAt: Scalars['Float']['output']; + message: Scalars['String']['output']; + startedAt: Scalars['Float']['output']; +}; + +export type BackgroundAssetWipeInProgress = { + __typename: 'BackgroundAssetWipeInProgress'; + startedAt: Scalars['Float']['output']; +}; + +export type BackgroundAssetWipeMutationResult = + | AssetNotFoundError + | AssetWipeInProgress + | PythonError + | UnauthorizedError + | UnsupportedOperationError; + +export type BackgroundAssetWipeStatus = + | BackgroundAssetWipeFailed + | BackgroundAssetWipeInProgress + | BackgroundAssetWipeSuccess; + +export type BackgroundAssetWipeSuccess = { + __typename: 'BackgroundAssetWipeSuccess'; + completedAt: Scalars['Float']['output']; + startedAt: Scalars['Float']['output']; +}; + export type BoolMetadataEntry = MetadataEntry & { __typename: 'BoolMetadataEntry'; boolValue: Maybe; @@ -2619,6 +2654,7 @@ export type MultiPartitionStatuses = { export type Mutation = { __typename: 'Mutation'; addDynamicPartition: AddDynamicPartitionResult; + backgroundWipeAssets: BackgroundAssetWipeMutationResult; cancelPartitionBackfill: CancelBackfillResult; deleteConcurrencyLimit: Scalars['Boolean']['output']; deleteDynamicPartitions: DeleteDynamicPartitionsResult; @@ -2663,6 +2699,10 @@ export type MutationAddDynamicPartitionArgs = { repositorySelector: RepositorySelector; }; +export type MutationBackgroundWipeAssetsArgs = { + assetPartitionRanges: Array; +}; + export type MutationCancelPartitionBackfillArgs = { backfillId: Scalars['String']['input']; }; @@ -3750,6 +3790,7 @@ export type Query = { autoMaterializeAssetEvaluationsOrError: Maybe; autoMaterializeEvaluationsForEvaluationId: Maybe; autoMaterializeTicks: Array; + backgroundAssetWipeStatus: BackgroundAssetWipeStatus; canBulkTerminate: Scalars['Boolean']['output']; capturedLogs: CapturedLogs; capturedLogsMetadata: CapturedLogsMetadata; @@ -3881,6 +3922,10 @@ export type QueryAutoMaterializeTicksArgs = { statuses?: InputMaybe>; }; +export type QueryBackgroundAssetWipeStatusArgs = { + workToken: Scalars['String']['input']; +}; + export type QueryCapturedLogsArgs = { cursor?: InputMaybe; limit?: InputMaybe; @@ -6877,6 +6922,19 @@ export const buildAssetSelection = ( }; }; +export const buildAssetWipeInProgress = ( + overrides?: Partial, + _relationshipsToOmit: Set = new Set(), +): {__typename: 'AssetWipeInProgress'} & AssetWipeInProgress => { + const relationshipsToOmit: Set = new Set(_relationshipsToOmit); + relationshipsToOmit.add('AssetWipeInProgress'); + return { + __typename: 'AssetWipeInProgress', + workToken: + overrides && overrides.hasOwnProperty('workToken') ? overrides.workToken! : 'quaerat', + }; +}; + export const buildAssetWipeSuccess = ( overrides?: Partial, _relationshipsToOmit: Set = new Set(), @@ -7116,6 +7174,46 @@ export const buildBackfillPolicy = ( }; }; +export const buildBackgroundAssetWipeFailed = ( + overrides?: Partial, + _relationshipsToOmit: Set = new Set(), +): {__typename: 'BackgroundAssetWipeFailed'} & BackgroundAssetWipeFailed => { + const relationshipsToOmit: Set = new Set(_relationshipsToOmit); + relationshipsToOmit.add('BackgroundAssetWipeFailed'); + return { + __typename: 'BackgroundAssetWipeFailed', + failedAt: overrides && overrides.hasOwnProperty('failedAt') ? overrides.failedAt! : 2.87, + message: overrides && overrides.hasOwnProperty('message') ? overrides.message! : 'dicta', + startedAt: overrides && overrides.hasOwnProperty('startedAt') ? overrides.startedAt! : 4.27, + }; +}; + +export const buildBackgroundAssetWipeInProgress = ( + overrides?: Partial, + _relationshipsToOmit: Set = new Set(), +): {__typename: 'BackgroundAssetWipeInProgress'} & BackgroundAssetWipeInProgress => { + const relationshipsToOmit: Set = new Set(_relationshipsToOmit); + relationshipsToOmit.add('BackgroundAssetWipeInProgress'); + return { + __typename: 'BackgroundAssetWipeInProgress', + startedAt: overrides && overrides.hasOwnProperty('startedAt') ? overrides.startedAt! : 0.47, + }; +}; + +export const buildBackgroundAssetWipeSuccess = ( + overrides?: Partial, + _relationshipsToOmit: Set = new Set(), +): {__typename: 'BackgroundAssetWipeSuccess'} & BackgroundAssetWipeSuccess => { + const relationshipsToOmit: Set = new Set(_relationshipsToOmit); + relationshipsToOmit.add('BackgroundAssetWipeSuccess'); + return { + __typename: 'BackgroundAssetWipeSuccess', + completedAt: + overrides && overrides.hasOwnProperty('completedAt') ? overrides.completedAt! : 3.54, + startedAt: overrides && overrides.hasOwnProperty('startedAt') ? overrides.startedAt! : 8.55, + }; +}; + export const buildBoolMetadataEntry = ( overrides?: Partial, _relationshipsToOmit: Set = new Set(), @@ -10235,6 +10333,12 @@ export const buildMutation = ( : relationshipsToOmit.has('AddDynamicPartitionSuccess') ? ({} as AddDynamicPartitionSuccess) : buildAddDynamicPartitionSuccess({}, relationshipsToOmit), + backgroundWipeAssets: + overrides && overrides.hasOwnProperty('backgroundWipeAssets') + ? overrides.backgroundWipeAssets! + : relationshipsToOmit.has('AssetNotFoundError') + ? ({} as AssetNotFoundError) + : buildAssetNotFoundError({}, relationshipsToOmit), cancelPartitionBackfill: overrides && overrides.hasOwnProperty('cancelPartitionBackfill') ? overrides.cancelPartitionBackfill! @@ -12152,6 +12256,12 @@ export const buildQuery = ( overrides && overrides.hasOwnProperty('autoMaterializeTicks') ? overrides.autoMaterializeTicks! : [], + backgroundAssetWipeStatus: + overrides && overrides.hasOwnProperty('backgroundAssetWipeStatus') + ? overrides.backgroundAssetWipeStatus! + : relationshipsToOmit.has('BackgroundAssetWipeFailed') + ? ({} as BackgroundAssetWipeFailed) + : buildBackgroundAssetWipeFailed({}, relationshipsToOmit), canBulkTerminate: overrides && overrides.hasOwnProperty('canBulkTerminate') ? overrides.canBulkTerminate! diff --git a/python_modules/dagster-graphql/dagster_graphql/implementation/execution/__init__.py b/python_modules/dagster-graphql/dagster_graphql/implementation/execution/__init__.py index 83d1c3af2d35a..69fb53d85f2b8 100644 --- a/python_modules/dagster-graphql/dagster_graphql/implementation/execution/__init__.py +++ b/python_modules/dagster-graphql/dagster_graphql/implementation/execution/__init__.py @@ -16,7 +16,12 @@ # re-exports import dagster._check as check from dagster._annotations import deprecated -from dagster._core.definitions.events import AssetKey, AssetPartitionWipeRange +from dagster._core.definitions.events import ( + AssetKey, + AssetPartitionWipeRange, + AssetWipeWorkItem, + AssetWipeWorkItemPartitionKeys, +) from dagster._core.events import ( AssetMaterialization, AssetObservation, @@ -57,6 +62,7 @@ GraphenePipelineRunLogsSubscriptionSuccess, ) from dagster_graphql.schema.roots.mutation import ( + GrapheneAssetWipeInProgress, GrapheneAssetWipeSuccess, GrapheneDeletePipelineRunSuccess, GrapheneReportRunlessAssetEventsSuccess, @@ -337,11 +343,38 @@ def _enqueue(new_event): subscription.dispose() +async def background_wipe_assets( + graphene_info: "ResolveInfo", asset_partition_ranges: Sequence[AssetPartitionWipeRange] +) -> Union[ + "GrapheneAssetWipeInProgress", + "GrapheneAssetNotFoundError", +]: + instance = graphene_info.context.instance + from dagster_graphql.schema.errors import GrapheneAssetNotFoundError + from dagster_graphql.schema.roots.mutation import GrapheneAssetWipeInProgress + + try: + work_item = get_asset_wipe_work(graphene_info, instance, asset_partition_ranges) + except AssetNotFoundError as e: + return GrapheneAssetNotFoundError(asset_key=e.asset_key) + + work_token = instance.background_asset_wipe(work_item) + return GrapheneAssetWipeInProgress(workToken=work_token) + + +class AssetNotFoundError(BaseException): + def __init__(self, asset_key: AssetKey): + self.asset_key = asset_key + + def wipe_assets( graphene_info: "ResolveInfo", asset_partition_ranges: Sequence[AssetPartitionWipeRange] ) -> Union[ - "GrapheneAssetWipeSuccess", "GrapheneUnsupportedOperationError", "GrapheneAssetNotFoundError" + "GrapheneAssetWipeSuccess", + "GrapheneUnsupportedOperationError", + "GrapheneAssetNotFoundError", ]: + instance = graphene_info.context.instance from dagster_graphql.schema.backfill import GrapheneAssetPartitionRange from dagster_graphql.schema.errors import ( GrapheneAssetNotFoundError, @@ -349,37 +382,61 @@ def wipe_assets( ) from dagster_graphql.schema.roots.mutation import GrapheneAssetWipeSuccess - instance = graphene_info.context.instance + try: + work_item = get_asset_wipe_work(graphene_info, instance, asset_partition_ranges) + except AssetNotFoundError as e: + return GrapheneAssetNotFoundError(asset_key=e.asset_key) + + try: + do_asset_wipe_work(instance, work_item) + # NotImplementedError will be thrown if the underlying EventLogStorage does not support + # partitioned asset wipe. + except NotImplementedError: + return GrapheneUnsupportedOperationError("Partitioned asset wipe is not supported yet.") + + result_ranges = [ + GrapheneAssetPartitionRange(asset_key=apr.asset_key, partition_range=apr.partition_range) + for apr in asset_partition_ranges + ] + return GrapheneAssetWipeSuccess(assetPartitionRanges=result_ranges) + + +def do_asset_wipe_work(instance: DagsterInstance, work_item: AssetWipeWorkItem) -> None: + for asset_partition_keys in work_item.asset_partition_keys: + instance.wipe_asset_partitions( + asset_partition_keys.asset_key, asset_partition_keys.partition_keys + ) + + instance.wipe_assets(work_item.assets) + + +def get_asset_wipe_work( + graphene_info: "ResolveInfo", + instance: DagsterInstance, + asset_partition_ranges: Sequence[AssetPartitionWipeRange], +) -> AssetWipeWorkItem: whole_assets_to_wipe: List[AssetKey] = [] + asset_partitions_to_wipe: List[AssetWipeWorkItemPartitionKeys] = [] for apr in asset_partition_ranges: if apr.partition_range is None: whole_assets_to_wipe.append(apr.asset_key) else: if apr.asset_key not in graphene_info.context.asset_graph.asset_node_snaps_by_key: - return GrapheneAssetNotFoundError(asset_key=apr.asset_key) + raise AssetNotFoundError(asset_key=apr.asset_key) node = graphene_info.context.asset_graph.asset_node_snaps_by_key[apr.asset_key] partitions_def = check.not_none(node.partitions).get_partitions_definition() partition_keys = partitions_def.get_partition_keys_in_range( apr.partition_range, dynamic_partitions_store=instance ) - try: - instance.wipe_asset_partitions(apr.asset_key, partition_keys) - - # NotImplementedError will be thrown if the underlying EventLogStorage does not support - # partitioned asset wipe. - except NotImplementedError: - return GrapheneUnsupportedOperationError( - "Partitioned asset wipe is not supported yet." + asset_partitions_to_wipe.append( + AssetWipeWorkItemPartitionKeys( + asset_key=apr.asset_key, partition_keys=partition_keys ) - - instance.wipe_assets(whole_assets_to_wipe) - - result_ranges = [ - GrapheneAssetPartitionRange(asset_key=apr.asset_key, partition_range=apr.partition_range) - for apr in asset_partition_ranges - ] - return GrapheneAssetWipeSuccess(assetPartitionRanges=result_ranges) + ) + return AssetWipeWorkItem( + assets=whole_assets_to_wipe, asset_partition_keys=asset_partitions_to_wipe + ) def create_asset_event( diff --git a/python_modules/dagster-graphql/dagster_graphql/schema/roots/assets.py b/python_modules/dagster-graphql/dagster_graphql/schema/roots/assets.py index 726c25ab89155..92c644e431e86 100644 --- a/python_modules/dagster-graphql/dagster_graphql/schema/roots/assets.py +++ b/python_modules/dagster-graphql/dagster_graphql/schema/roots/assets.py @@ -23,3 +23,37 @@ class GrapheneAssetOrError(graphene.Union): class Meta: types = (GrapheneAsset, GrapheneAssetNotFoundError) name = "AssetOrError" + + +class GrapheneBackgroundAssetWipeSuccess(graphene.ObjectType): + startedAt = graphene.NonNull(graphene.Float) + completedAt = graphene.NonNull(graphene.Float) + + class Meta: + name = "BackgroundAssetWipeSuccess" + + +class GrapheneBackgroundAssetWipeInProgress(graphene.ObjectType): + startedAt = graphene.NonNull(graphene.Float) + + class Meta: + name = "BackgroundAssetWipeInProgress" + + +class GrapheneBackgroundAssetWipeFailed(graphene.ObjectType): + startedAt = graphene.NonNull(graphene.Float) + failedAt = graphene.NonNull(graphene.Float) + message = graphene.NonNull(graphene.String) + + class Meta: + name = "BackgroundAssetWipeFailed" + + +class GrapheneBackgroundAssetWipeStatus(graphene.Union): + class Meta: + types = ( + GrapheneBackgroundAssetWipeSuccess, + GrapheneBackgroundAssetWipeInProgress, + GrapheneBackgroundAssetWipeFailed, + ) + name = "BackgroundAssetWipeStatus" diff --git a/python_modules/dagster-graphql/dagster_graphql/schema/roots/mutation.py b/python_modules/dagster-graphql/dagster_graphql/schema/roots/mutation.py index 687d1fd70ba60..be0b6a32efbd2 100644 --- a/python_modules/dagster-graphql/dagster_graphql/schema/roots/mutation.py +++ b/python_modules/dagster-graphql/dagster_graphql/schema/roots/mutation.py @@ -10,6 +10,7 @@ from dagster._daemon.asset_daemon import set_auto_materialize_paused from dagster_graphql.implementation.execution import ( + background_wipe_assets, delete_pipeline_run, report_runless_asset_events, terminate_pipeline_execution, @@ -745,6 +746,15 @@ class Meta: name = "AssetWipeSuccess" +class GrapheneAssetWipeInProgress(graphene.ObjectType): + """Output indicating that asset history deletion is in progress.""" + + workToken = graphene.NonNull(graphene.String) + + class Meta: + name = "AssetWipeInProgress" + + class GrapheneAssetWipeMutationResult(graphene.Union): """The output from deleting asset history.""" @@ -759,6 +769,53 @@ class Meta: name = "AssetWipeMutationResult" +class GrapheneBackgroundAssetWipeMutationResult(graphene.Union): + """The output from deleting asset history.""" + + class Meta: + types = ( + GrapheneAssetNotFoundError, + GrapheneUnauthorizedError, + GraphenePythonError, + GrapheneUnsupportedOperationError, + GrapheneAssetWipeInProgress, + ) + name = "BackgroundAssetWipeMutationResult" + + +class GrapheneBackgroundAssetWipeMutation(graphene.Mutation): + """Deletes asset history from storage in the background, returning instead a work + token that can be used to check the progress of the delete. + """ + + Output = graphene.NonNull(GrapheneBackgroundAssetWipeMutationResult) + + class Arguments: + assetPartitionRanges = graphene.Argument(non_null_list(GraphenePartitionsByAssetSelector)) + + class Meta: + name = "BackgroundAssetWipeMutation" + + @capture_error + @check_permission(Permissions.WIPE_ASSETS) + async def mutate( + self, + graphene_info: ResolveInfo, + assetPartitionRanges: Sequence[GraphenePartitionsByAssetSelector], + ) -> GrapheneBackgroundAssetWipeMutationResult: + normalized_ranges = [ + AssetPartitionWipeRange( + AssetKey.from_graphql_input(ap.assetKey), + PartitionKeyRange(start=ap.partitions.range.start, end=ap.partitions.range.end) + if ap.partitions + else None, + ) + for ap in assetPartitionRanges + ] + + return await background_wipe_assets(graphene_info, normalized_ranges) + + class GrapheneAssetWipeMutation(graphene.Mutation): """Deletes asset history from storage.""" @@ -1039,6 +1096,7 @@ class Meta: reloadWorkspace = GrapheneReloadWorkspaceMutation.Field() shutdownRepositoryLocation = GrapheneShutdownRepositoryLocationMutation.Field() wipeAssets = GrapheneAssetWipeMutation.Field() + backgroundWipeAssets = GrapheneBackgroundAssetWipeMutation.Field() reportRunlessAssetEvents = GrapheneReportRunlessAssetEventsMutation.Field() launchPartitionBackfill = GrapheneLaunchBackfillMutation.Field() resumePartitionBackfill = GrapheneResumeBackfillMutation.Field() diff --git a/python_modules/dagster-graphql/dagster_graphql/schema/roots/query.py b/python_modules/dagster-graphql/dagster_graphql/schema/roots/query.py index 48a244bee6457..b0a1324c6fe27 100644 --- a/python_modules/dagster-graphql/dagster_graphql/schema/roots/query.py +++ b/python_modules/dagster-graphql/dagster_graphql/schema/roots/query.py @@ -1,10 +1,11 @@ +from datetime import datetime from typing import Any, Dict, List, Mapping, Optional, Sequence, cast import dagster._check as check import graphene from dagster import AssetCheckKey from dagster._core.definitions.asset_graph_differ import AssetGraphDiffer -from dagster._core.definitions.events import AssetKey +from dagster._core.definitions.events import AssetKey, BackgroundAssetWipeStatus from dagster._core.definitions.partition import CachingDynamicPartitionsLoader from dagster._core.definitions.remote_asset_graph import RemoteAssetGraph from dagster._core.definitions.selector import ( @@ -175,7 +176,14 @@ GrapheneResourceDetailsListOrError, GrapheneResourceDetailsOrError, ) -from dagster_graphql.schema.roots.assets import GrapheneAssetOrError, GrapheneAssetsOrError +from dagster_graphql.schema.roots.assets import ( + GrapheneAssetOrError, + GrapheneAssetsOrError, + GrapheneBackgroundAssetWipeFailed, + GrapheneBackgroundAssetWipeInProgress, + GrapheneBackgroundAssetWipeStatus, + GrapheneBackgroundAssetWipeSuccess, +) from dagster_graphql.schema.roots.execution_plan import GrapheneExecutionPlanOrError from dagster_graphql.schema.roots.pipeline import GrapheneGraphOrError, GraphenePipelineOrError from dagster_graphql.schema.run_config import GrapheneRunConfigSchemaOrError @@ -512,6 +520,12 @@ class Meta: description="Fetch the partitions that would be targeted by a backfill, given the root partitions targeted.", ) + backgroundAssetWipeStatus = graphene.Field( + graphene.NonNull(GrapheneBackgroundAssetWipeStatus), + workToken=graphene.Argument(graphene.NonNull(graphene.String)), + description="Retrieve the status of the given background asset wipe.", + ) + partitionBackfillsOrError = graphene.Field( graphene.NonNull(GraphenePartitionBackfillsOrError), status=graphene.Argument(GrapheneBulkActionStatus), @@ -1175,6 +1189,29 @@ def resolve_partitionBackfillsOrError( filters=filters.to_selector() if filters else None, ) + @capture_error + async def resolve_backgroundAssetWipeStatus( + self, + graphene_info: ResolveInfo, + workToken: str, + ): + instance = graphene_info.context.instance + status = instance.get_background_asset_wipe_status(workToken) + if status == BackgroundAssetWipeStatus.SUCCESS: + return GrapheneBackgroundAssetWipeSuccess( + startedAt=datetime.now().timestamp(), completedAt=datetime.now().timestamp() + ) + elif status == BackgroundAssetWipeStatus.IN_PROGRESS: + return GrapheneBackgroundAssetWipeInProgress(startedAt=datetime.now().timestamp()) + elif status == BackgroundAssetWipeStatus.FAILED: + return GrapheneBackgroundAssetWipeFailed( + startedAt=datetime.now().timestamp(), + completedAt=datetime.now().timestamp(), + message="oh no", + ) + else: + raise Exception(f"unexpected asset wipe status: {status}") + def resolve_assetBackfillPreview( self, graphene_info: ResolveInfo, params: GrapheneAssetBackfillPreviewParams ) -> Sequence[GrapheneAssetPartitions]: diff --git a/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_assets.py b/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_assets.py index 0901f0b9af9da..98451d06f931d 100644 --- a/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_assets.py +++ b/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_assets.py @@ -99,6 +99,33 @@ """ +BACKGROUND_WIPE_ASSETS = """ + mutation BackgroundAssetKeyWipe($assetPartitionRanges: [PartitionsByAssetSelector!]!) { + backgroundWipeAssets(assetPartitionRanges: $assetPartitionRanges) { + __typename + ... on AssetWipeInProgress { + workToken + } + } + } +""" + + +BACKGROUND_ASSET_WIPE_STATUS = """ + query BackgroundAssetKeyWipeStatus($workToken: String!) { + backgroundAssetWipeStatus(workToken: $workToken) { + __typename + ... on BackgroundAssetWipeSuccess { + completedAt + } + ... on BackgroundAssetWipeInProgress { + startedAt + } + } + } +""" + + WIPE_ASSETS = """ mutation AssetKeyWipe($assetPartitionRanges: [PartitionsByAssetSelector!]!) { wipeAssets(assetPartitionRanges: $assetPartitionRanges) { diff --git a/python_modules/dagster-test/dagster_test/fixtures/docker_compose.py b/python_modules/dagster-test/dagster_test/fixtures/docker_compose.py index 168780d62a6cd..82d8fa7e236c8 100644 --- a/python_modules/dagster-test/dagster_test/fixtures/docker_compose.py +++ b/python_modules/dagster-test/dagster_test/fixtures/docker_compose.py @@ -5,6 +5,7 @@ from contextlib import contextmanager import pytest +import yaml from dagster_test.fixtures.utils import BUILDKITE @@ -20,6 +21,7 @@ def docker_compose_cm( ): if not network_name: network_name = network_name_from_yml(docker_compose_yml) + try: try: docker_compose_up( @@ -192,14 +194,24 @@ def buildkite_hostnames_cm(network): disconnect_container_from_network(container, network) -def default_docker_compose_yml(default_directory): +def default_docker_compose_yml(default_directory) -> str: if os.path.isfile("docker-compose.yml"): return os.path.join(os.getcwd(), "docker-compose.yml") else: return os.path.join(default_directory, "docker-compose.yml") -def network_name_from_yml(docker_compose_yml): - dirname = os.path.dirname(docker_compose_yml) - basename = os.path.basename(dirname) - return basename + "_default" +def network_name_from_yml(docker_compose_yml) -> str: + with open(docker_compose_yml) as f: + config = yaml.safe_load(f) + if "name" in config: + name = config["name"] + else: + dirname = os.path.dirname(docker_compose_yml) + name = os.path.basename(dirname) + if "networks" in config: + network_name = next(iter(config["networks"].keys())) + else: + network_name = "default" + + return f"{name}_{network_name}" diff --git a/python_modules/dagster/dagster/_core/definitions/events.py b/python_modules/dagster/dagster/_core/definitions/events.py index f5ceb9bf1b0f9..a47eebef8e6c4 100644 --- a/python_modules/dagster/dagster/_core/definitions/events.py +++ b/python_modules/dagster/dagster/_core/definitions/events.py @@ -42,6 +42,7 @@ from dagster._core.definitions.partition_key_range import PartitionKeyRange from dagster._core.definitions.utils import DEFAULT_OUTPUT, check_valid_name from dagster._core.storage.tags import MULTIDIMENSIONAL_PARTITION_PREFIX, REPORTING_USER_TAG +from dagster._record import record from dagster._serdes import whitelist_for_serdes from dagster._serdes.serdes import NamedTupleSerializer @@ -58,6 +59,17 @@ class AssetKeyPartitionKey(NamedTuple): partition_key: Optional[str] = None +# This is a "work token" associated with the background process responsible +# for wiping the asset +BackgroundAssetWipeWorkToken = str + + +class BackgroundAssetWipeStatus(Enum): + SUCCESS = "SUCCESS" + IN_PROGRESS = "IN_PROGRESS" + FAILED = "FAILED" + + # This is currently used only for the asset partition wipe codepath. In the future, we can rename # to AssetPartitionRange or similar for more general use. class AssetPartitionWipeRange(NamedTuple): @@ -67,6 +79,20 @@ class AssetPartitionWipeRange(NamedTuple): partition_range: Optional[PartitionKeyRange] +@whitelist_for_serdes +@record +class AssetWipeWorkItemPartitionKeys: + asset_key: AssetKey + partition_keys: Sequence[str] + + +@whitelist_for_serdes +@record +class AssetWipeWorkItem: + assets: List[AssetKey] + asset_partition_keys: List[AssetWipeWorkItemPartitionKeys] + + DynamicAssetKey = Callable[["OutputContext"], Optional[AssetKey]] diff --git a/python_modules/dagster/dagster/_core/instance/__init__.py b/python_modules/dagster/dagster/_core/instance/__init__.py index fb42ffe3b8c2a..d469a8549353c 100644 --- a/python_modules/dagster/dagster/_core/instance/__init__.py +++ b/python_modules/dagster/dagster/_core/instance/__init__.py @@ -38,7 +38,13 @@ AssetCheckEvaluationPlanned, ) from dagster._core.definitions.data_version import extract_data_provenance_from_entry -from dagster._core.definitions.events import AssetKey, AssetObservation +from dagster._core.definitions.events import ( + AssetKey, + AssetObservation, + AssetWipeWorkItem, + BackgroundAssetWipeStatus, + BackgroundAssetWipeWorkToken, +) from dagster._core.definitions.partition_key_range import PartitionKeyRange from dagster._core.errors import ( DagsterHomeNotSetError, @@ -3304,5 +3310,14 @@ def get_asset_check_support(self) -> "AssetCheckInstanceSupport": def backfill_log_storage_enabled(self) -> bool: return False + def background_asset_wipe( + self, + work_item: AssetWipeWorkItem, + ) -> BackgroundAssetWipeWorkToken: + raise NotImplementedError() + + def get_background_asset_wipe_status(self, work_token: str) -> BackgroundAssetWipeStatus: + raise NotImplementedError() + def da_request_backfills(self) -> bool: return False diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_dbt_project.py b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_dbt_project.py index 5386816b507b4..323154de35b40 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_dbt_project.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_dbt_project.py @@ -13,9 +13,8 @@ defs_from_components, ) from dagster_components.impls.dbt_project import DbtProjectComponent -from dagster_dbt import DbtProject - from dagster_components_tests.utils import assert_assets, get_asset_keys, script_load_context +from dagster_dbt import DbtProject STUB_LOCATION_PATH = Path(__file__).parent.parent / "stub_code_locations" / "dbt_project_location" COMPONENT_RELPATH = "components/jaffle_shop_dbt"