From df5abbebd348407c5323b3008b851ef427fa3bc7 Mon Sep 17 00:00:00 2001 From: Mathieu Larose Date: Thu, 26 Dec 2024 10:59:09 -0500 Subject: [PATCH 1/2] [computelogmanager] add fields to pass uri, path or shell cmd --- .../dagster-ui/packages/ui-core/client.json | 4 +- .../ui-core/src/graphql/schema.graphql | 4 + .../packages/ui-core/src/graphql/types.ts | 16 ++++ .../ui-core/src/runs/CapturedLogPanel.tsx | 95 +++++++++++++++---- .../packages/ui-core/src/runs/LogsRow.tsx | 4 + .../ui-core/src/runs/RunMetadataProvider.tsx | 12 +++ .../src/runs/types/LogsProvider.types.ts | 16 +++- .../ui-core/src/runs/types/LogsRow.types.ts | 4 + ...LogsScrollingTableMessageFragment.types.ts | 4 + .../src/runs/types/RunFragments.types.ts | 4 + .../runs/types/RunMetadataProvider.types.ts | 4 + .../dagster_graphql/implementation/events.py | 4 + .../implementation/fetch_logs.py | 2 + .../dagster_graphql/schema/logs/events.py | 4 + .../dagster/dagster/_core/events/__init__.py | 16 ++++ .../cloud_storage_compute_log_manager.py | 14 +++ .../_core/storage/compute_log_manager.py | 24 +++++ 17 files changed, 209 insertions(+), 22 deletions(-) diff --git a/js_modules/dagster-ui/packages/ui-core/client.json b/js_modules/dagster-ui/packages/ui-core/client.json index 8e9e8ae0c4b40..92f87b1bfecf1 100644 --- a/js_modules/dagster-ui/packages/ui-core/client.json +++ b/js_modules/dagster-ui/packages/ui-core/client.json @@ -115,8 +115,8 @@ "CapturedLogsSubscription": "fa5e55b59e9d8632ae71a8387c54230ba71e6f57849a974225ba039808acfa93", "CapturedLogsMetadataQuery": "b59ada7585593473002a7b044f09daa85f160445cbc9a4e8ffe0b46d51875cb1", "CapturedLogsQuery": "872b617b4f33ee5f6feeba9a4c76ec986fca357695a114e3d7b63172e4600b57", - "PipelineRunLogsSubscription": "def2c2dc20d6b8640de7f4104ea03a58c8721cb5a632282f8e2d7489ab205280", - "RunLogsQuery": "0f1a38047ec63b668879f68aff67dc13d7a2b241c2f506d0109968897e01f385", + "PipelineRunLogsSubscription": "e2c0d006ce476a1059e591582520e88b431f413fc99401c6eafbb3844a7abe93", + "RunLogsQuery": "c5ce527d5df7e8771f351f4d50fb06614c39696833f9dd02a4f7a3832bd60dac", "QueuedRunCriteriaQuery": "da19aeed8a0a7e6f47619c6ba9efd721345481d8f08223282ea774e468400f21", "QueueDaemonStatusQuery": "aa51c596ee907346e60e2fe173bba10ae2ead067d45109225a2cd400a2278841", "PipelineEnvironmentQuery": "3b668b028997fb35b17b4d8a90a18b78dd8a70910f2c12aac63065c0584e3a10", diff --git a/js_modules/dagster-ui/packages/ui-core/src/graphql/schema.graphql b/js_modules/dagster-ui/packages/ui-core/src/graphql/schema.graphql index 2f4c3193df0c9..dde1512f15498 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/graphql/schema.graphql +++ b/js_modules/dagster-ui/packages/ui-core/src/graphql/schema.graphql @@ -1239,6 +1239,10 @@ type LogsCapturedEvent implements MessageEvent { externalUrl: String externalStdoutUrl: String externalStderrUrl: String + stderrUriOrPath: String + stdoutUriOrPath: String + stdoutShellCmd: String + stderrShellCmd: String pid: Int logKey: String! } diff --git a/js_modules/dagster-ui/packages/ui-core/src/graphql/types.ts b/js_modules/dagster-ui/packages/ui-core/src/graphql/types.ts index ed57441d95d09..001f1459b8f2d 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/graphql/types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/graphql/types.ts @@ -2432,6 +2432,10 @@ export type LogsCapturedEvent = MessageEvent & { pid: Maybe; runId: Scalars['String']['output']; solidHandleID: Maybe; + stderrShellCmd: Maybe; + stderrUriOrPath: Maybe; + stdoutShellCmd: Maybe; + stdoutUriOrPath: Maybe; stepKey: Maybe; stepKeys: Maybe>; timestamp: Scalars['String']['output']; @@ -9820,6 +9824,18 @@ export const buildLogsCapturedEvent = ( overrides && overrides.hasOwnProperty('solidHandleID') ? overrides.solidHandleID! : 'assumenda', + stderrShellCmd: + overrides && overrides.hasOwnProperty('stderrShellCmd') ? overrides.stderrShellCmd! : 'non', + stderrUriOrPath: + overrides && overrides.hasOwnProperty('stderrUriOrPath') + ? overrides.stderrUriOrPath! + : 'amet', + stdoutShellCmd: + overrides && overrides.hasOwnProperty('stdoutShellCmd') ? overrides.stdoutShellCmd! : 'eos', + stdoutUriOrPath: + overrides && overrides.hasOwnProperty('stdoutUriOrPath') + ? overrides.stdoutUriOrPath! + : 'recusandae', stepKey: overrides && overrides.hasOwnProperty('stepKey') ? overrides.stepKey! : 'quia', stepKeys: overrides && overrides.hasOwnProperty('stepKeys') ? overrides.stepKeys! : [], timestamp: overrides && overrides.hasOwnProperty('timestamp') ? overrides.timestamp! : 'et', diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/CapturedLogPanel.tsx b/js_modules/dagster-ui/packages/ui-core/src/runs/CapturedLogPanel.tsx index 4655da40ef8bc..68f7878344773 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/CapturedLogPanel.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/CapturedLogPanel.tsx @@ -1,4 +1,4 @@ -import {Box, Colors, Icon} from '@dagster-io/ui-components'; +import {Box, Colors, Icon, Tooltip, UnstyledButton} from '@dagster-io/ui-components'; import * as React from 'react'; import {RawLogContent} from './RawLogContent'; @@ -14,7 +14,9 @@ import { CapturedLogsSubscriptionVariables, } from './types/CapturedLogPanel.types'; import {AppContext} from '../app/AppContext'; +import {showSharedToaster} from '../app/DomUtils'; import {WebSocketContext} from '../app/WebSocketProvider'; +import {useCopyToClipboard} from '../app/browser'; interface CapturedLogProps { logKey: string[]; @@ -33,28 +35,85 @@ export const CapturedOrExternalLogPanel = React.memo( (props.visibleIOType === 'stdout' ? logCaptureInfo.externalStdoutUrl : logCaptureInfo.externalStderrUrl); - if (externalUrl) { + const uriOrPath = + logCaptureInfo && + (props.visibleIOType === 'stdout' + ? logCaptureInfo.stdoutUriOrPath + : logCaptureInfo.stderrUriOrPath); + const shellCmd = + logCaptureInfo && + (props.visibleIOType === 'stdout' + ? logCaptureInfo.stdoutShellCmd + : logCaptureInfo.stderrShellCmd); + + const copy = useCopyToClipboard(); + const onClickFn = async (key: string, value: string | undefined) => { + if (!value) { + return; + } + copy(value); + await showSharedToaster({ + intent: 'success', + icon: 'done', + message: `${key} string copied!`, + }); + }; + const onClickExternalUri = async () => onClickFn('log artefact URI', uriOrPath); + const onClickShellCmd = async () => onClickFn('shell command', shellCmd); + + if (externalUrl || uriOrPath || shellCmd) { return ( - View logs at - - {externalUrl} - - + {externalUrl ? ( +
+ View logs at + + {externalUrl} + + +
+ ) : undefined} + + {uriOrPath ? ( +
+ Logs artefact URI: + + {uriOrPath} + +
+ ) : undefined} + + {shellCmd ? ( +
+ Shell command to download logs: + + {shellCmd} + +
+ ) : undefined} ); } diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/LogsRow.tsx b/js_modules/dagster-ui/packages/ui-core/src/runs/LogsRow.tsx index 32e9bb48044da..d48cdfca8b888 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/LogsRow.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/LogsRow.tsx @@ -197,6 +197,10 @@ export const LOGS_ROW_STRUCTURED_FRAGMENT = gql` externalUrl externalStdoutUrl externalStderrUrl + stdoutUriOrPath + stderrUriOrPath + stdoutShellCmd + stderrShellCmd } ... on AssetCheckEvaluationEvent { evaluation { diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/RunMetadataProvider.tsx b/js_modules/dagster-ui/packages/ui-core/src/runs/RunMetadataProvider.tsx index a3d495d021e7e..37469cf96c680 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/RunMetadataProvider.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/RunMetadataProvider.tsx @@ -73,6 +73,10 @@ export interface ILogCaptureInfo { pid?: string; externalStdoutUrl?: string; externalStderrUrl?: string; + stdoutUriOrPath?: string; + stderrUriOrPath?: string; + stdoutShellCmd?: string; + stderrShellCmd?: string; } export interface IRunMetadataDict { @@ -277,6 +281,10 @@ export function extractMetadataFromLogs( pid: String(log.pid), externalStdoutUrl: log.externalStdoutUrl || undefined, externalStderrUrl: log.externalStderrUrl || undefined, + stdoutUriOrPath: log.stdoutUriOrPath || undefined, + stderrUriOrPath: log.stderrUriOrPath || undefined, + stdoutShellCmd: log.stdoutShellCmd || undefined, + stderrShellCmd: log.stderrShellCmd || undefined, }; } @@ -454,6 +462,10 @@ export const RUN_METADATA_PROVIDER_MESSAGE_FRAGMENT = gql` pid externalStdoutUrl externalStderrUrl + stdoutUriOrPath + stderrUriOrPath + stdoutShellCmd + stderrShellCmd } } diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/types/LogsProvider.types.ts b/js_modules/dagster-ui/packages/ui-core/src/runs/types/LogsProvider.types.ts index bd9e99d3b0464..c6c1819025594 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/types/LogsProvider.types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/types/LogsProvider.types.ts @@ -1654,6 +1654,10 @@ export type PipelineRunLogsSubscription = { pid: number | null; externalStdoutUrl: string | null; externalStderrUrl: string | null; + stdoutUriOrPath: string | null; + stderrUriOrPath: string | null; + stdoutShellCmd: string | null; + stderrShellCmd: string | null; eventType: Types.DagsterEventType | null; externalUrl: string | null; } @@ -4996,6 +5000,10 @@ export type RunLogsSubscriptionSuccessFragment = { pid: number | null; externalStdoutUrl: string | null; externalStderrUrl: string | null; + stdoutUriOrPath: string | null; + stderrUriOrPath: string | null; + stdoutShellCmd: string | null; + stderrShellCmd: string | null; eventType: Types.DagsterEventType | null; externalUrl: string | null; } @@ -8347,6 +8355,10 @@ export type RunLogsQuery = { pid: number | null; externalStdoutUrl: string | null; externalStderrUrl: string | null; + stdoutUriOrPath: string | null; + stderrUriOrPath: string | null; + stdoutShellCmd: string | null; + stderrShellCmd: string | null; eventType: Types.DagsterEventType | null; externalUrl: string | null; } @@ -10084,6 +10096,6 @@ export type RunLogsQuery = { | {__typename: 'RunNotFoundError'}; }; -export const PipelineRunLogsSubscriptionVersion = 'def2c2dc20d6b8640de7f4104ea03a58c8721cb5a632282f8e2d7489ab205280'; +export const PipelineRunLogsSubscriptionVersion = 'e2c0d006ce476a1059e591582520e88b431f413fc99401c6eafbb3844a7abe93'; -export const RunLogsQueryVersion = '0f1a38047ec63b668879f68aff67dc13d7a2b241c2f506d0109968897e01f385'; +export const RunLogsQueryVersion = 'c5ce527d5df7e8771f351f4d50fb06614c39696833f9dd02a4f7a3832bd60dac'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/types/LogsRow.types.ts b/js_modules/dagster-ui/packages/ui-core/src/runs/types/LogsRow.types.ts index fa3ab85f02214..bc3ca6834a1f3 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/types/LogsRow.types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/types/LogsRow.types.ts @@ -1484,6 +1484,10 @@ export type LogsRowStructuredFragment_LogsCapturedEvent = { externalUrl: string | null; externalStdoutUrl: string | null; externalStderrUrl: string | null; + stdoutUriOrPath: string | null; + stderrUriOrPath: string | null; + stdoutShellCmd: string | null; + stderrShellCmd: string | null; }; export type LogsRowStructuredFragment_MaterializationEvent = { diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/types/LogsScrollingTableMessageFragment.types.ts b/js_modules/dagster-ui/packages/ui-core/src/runs/types/LogsScrollingTableMessageFragment.types.ts index 87127d7ac63cb..3450b86af4efc 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/types/LogsScrollingTableMessageFragment.types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/types/LogsScrollingTableMessageFragment.types.ts @@ -1484,6 +1484,10 @@ export type LogsScrollingTableMessageFragment_LogsCapturedEvent = { externalUrl: string | null; externalStdoutUrl: string | null; externalStderrUrl: string | null; + stdoutUriOrPath: string | null; + stderrUriOrPath: string | null; + stdoutShellCmd: string | null; + stderrShellCmd: string | null; }; export type LogsScrollingTableMessageFragment_MaterializationEvent = { diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunFragments.types.ts b/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunFragments.types.ts index e75aab9ccde18..b069987742ceb 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunFragments.types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunFragments.types.ts @@ -1532,6 +1532,10 @@ export type RunDagsterRunEventFragment_LogsCapturedEvent = { pid: number | null; externalStdoutUrl: string | null; externalStderrUrl: string | null; + stdoutUriOrPath: string | null; + stderrUriOrPath: string | null; + stdoutShellCmd: string | null; + stderrShellCmd: string | null; eventType: Types.DagsterEventType | null; externalUrl: string | null; }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunMetadataProvider.types.ts b/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunMetadataProvider.types.ts index 9c7d0454624ba..24a715fb8b8c1 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunMetadataProvider.types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunMetadataProvider.types.ts @@ -207,6 +207,10 @@ export type RunMetadataProviderMessageFragment_LogsCapturedEvent = { pid: number | null; externalStdoutUrl: string | null; externalStderrUrl: string | null; + stdoutUriOrPath: string | null; + stderrUriOrPath: string | null; + stdoutShellCmd: string | null; + stderrShellCmd: string | null; }; export type RunMetadataProviderMessageFragment_MaterializationEvent = { diff --git a/python_modules/dagster-graphql/dagster_graphql/implementation/events.py b/python_modules/dagster-graphql/dagster_graphql/implementation/events.py index 7e8fd1be4dca6..3267abc16278c 100644 --- a/python_modules/dagster-graphql/dagster_graphql/implementation/events.py +++ b/python_modules/dagster-graphql/dagster_graphql/implementation/events.py @@ -426,6 +426,10 @@ def from_dagster_event_record(event_record: EventLogEntry, pipeline_name: str) - externalUrl=data.external_url, externalStdoutUrl=data.external_stdout_url or data.external_url, externalStderrUrl=data.external_stderr_url or data.external_url, + stdoutUriOrPath=data.stdout_uri_or_path, + stderrUriOrPath=data.stderr_uri_or_path, + stdoutShellCmd=data.stdout_shell_cmd, + stderrShellCmd=data.stderr_shell_cmd, pid=dagster_event.pid, **basic_params, ) diff --git a/python_modules/dagster-graphql/dagster_graphql/implementation/fetch_logs.py b/python_modules/dagster-graphql/dagster_graphql/implementation/fetch_logs.py index bf267a1545b1b..a240ef194ef9d 100644 --- a/python_modules/dagster-graphql/dagster_graphql/implementation/fetch_logs.py +++ b/python_modules/dagster-graphql/dagster_graphql/implementation/fetch_logs.py @@ -17,4 +17,6 @@ def get_captured_log_metadata( stdoutLocation=metadata.stdout_location, stderrDownloadUrl=metadata.stderr_download_url, stderrLocation=metadata.stderr_location, + stdoutShellCmd=metadata.stdout_shell_cmd, + stderrShellCmd=metadata.stderr_shell_cmd, ) diff --git a/python_modules/dagster-graphql/dagster_graphql/schema/logs/events.py b/python_modules/dagster-graphql/dagster_graphql/schema/logs/events.py index 46a1dbc8bf0e2..d56626c4957b8 100644 --- a/python_modules/dagster-graphql/dagster_graphql/schema/logs/events.py +++ b/python_modules/dagster-graphql/dagster_graphql/schema/logs/events.py @@ -309,6 +309,10 @@ class Meta: externalUrl = graphene.String() externalStdoutUrl = graphene.String() externalStderrUrl = graphene.String() + stderrUriOrPath = graphene.String() + stdoutUriOrPath = graphene.String() + stdoutShellCmd = graphene.String() + stderrShellCmd = graphene.String() pid = graphene.Int() # legacy name for compute log file key... required for back-compat reasons, but has been # renamed to fileKey for newer versions of the Dagster UI diff --git a/python_modules/dagster/dagster/_core/events/__init__.py b/python_modules/dagster/dagster/_core/events/__init__.py index 0ec330df6f9f9..7795e68320e51 100644 --- a/python_modules/dagster/dagster/_core/events/__init__.py +++ b/python_modules/dagster/dagster/_core/events/__init__.py @@ -1493,8 +1493,12 @@ def capture_logs( event_specific_data=ComputeLogsCaptureData( step_keys=step_keys, file_key=file_key, + stdout_uri_or_path=log_context.stdout_uri_or_path, + stderr_uri_or_path=log_context.stderr_uri_or_path, external_stdout_url=log_context.external_stdout_url, external_stderr_url=log_context.external_stderr_url, + stdout_shell_cmd=log_context.stdout_shell_cmd, + stderr_shell_cmd=log_context.stderr_shell_cmd, external_url=log_context.external_url, ), ) @@ -1877,6 +1881,10 @@ class ComputeLogsCaptureData( ("external_url", Optional[str]), ("external_stdout_url", Optional[str]), ("external_stderr_url", Optional[str]), + ("stdout_uri_or_path", Optional[str]), + ("stderr_uri_or_path", Optional[str]), + ("stdout_shell_cmd", Optional[str]), + ("stderr_shell_cmd", Optional[str]), ], ) ): @@ -1887,6 +1895,10 @@ def __new__( external_url: Optional[str] = None, external_stdout_url: Optional[str] = None, external_stderr_url: Optional[str] = None, + stdout_uri_or_path: Optional[str] = None, + stderr_uri_or_path: Optional[str] = None, + stdout_shell_cmd: Optional[str] = None, + stderr_shell_cmd: Optional[str] = None, ): return super(ComputeLogsCaptureData, cls).__new__( cls, @@ -1895,6 +1907,10 @@ def __new__( external_url=check.opt_str_param(external_url, "external_url"), external_stdout_url=check.opt_str_param(external_stdout_url, "external_stdout_url"), external_stderr_url=check.opt_str_param(external_stderr_url, "external_stderr_url"), + stdout_uri_or_path=check.opt_str_param(stdout_uri_or_path, "stdout_uri_or_path"), + stderr_uri_or_path=check.opt_str_param(stderr_uri_or_path, "stderr_uri_or_path"), + stdout_shell_cmd=check.opt_str_param(stdout_shell_cmd, "stdout_shell_cmd"), + stderr_shell_cmd=check.opt_str_param(stderr_shell_cmd, "stderr_shell_cmd"), ) diff --git a/python_modules/dagster/dagster/_core/storage/cloud_storage_compute_log_manager.py b/python_modules/dagster/dagster/_core/storage/cloud_storage_compute_log_manager.py index 94cfbc1706f5c..0c561b214ab32 100644 --- a/python_modules/dagster/dagster/_core/storage/cloud_storage_compute_log_manager.py +++ b/python_modules/dagster/dagster/_core/storage/cloud_storage_compute_log_manager.py @@ -48,6 +48,16 @@ def delete_logs( def download_url_for_type(self, log_key: Sequence[str], io_type: ComputeIOType) -> str: """Calculates a download url given a log key and compute io type.""" + @abstractmethod + def uri_or_path_for_type(self, log_key: Sequence[str], io_type: ComputeIOType) -> Optional[str]: + """Calculates a download uri given a log key and compute io type.""" + return None + + @abstractmethod + def shell_command_for_type(self, log_key: Sequence[str], io_type: ComputeIOType) -> Optional[str]: + """Returns a shell command to download logs for a given log key and compute io type.""" + return None + @abstractmethod def display_path_for_type(self, log_key: Sequence[str], io_type: ComputeIOType) -> str: """Returns a display path given a log key and compute io type.""" @@ -127,6 +137,10 @@ def get_log_metadata(self, log_key: Sequence[str]) -> CapturedLogMetadata: stderr_location=self.display_path_for_type(log_key, ComputeIOType.STDERR), stdout_download_url=self.download_url_for_type(log_key, ComputeIOType.STDOUT), stderr_download_url=self.download_url_for_type(log_key, ComputeIOType.STDERR), + stdout_uri_or_path=self.uri_or_path_for_type(log_key, ComputeIOType.STDOUT), + stderr_uri_or_path=self.uri_or_path_for_type(log_key, ComputeIOType.STDERR), + stdout_shell_cmd=self.shell_command_for_type(log_key, ComputeIOType.STDOUT), + stderr_shell_cmd=self.shell_command_for_type(log_key, ComputeIOType.STDERR), ) def on_progress(self, log_key): diff --git a/python_modules/dagster/dagster/_core/storage/compute_log_manager.py b/python_modules/dagster/dagster/_core/storage/compute_log_manager.py index a737082743d88..227022ebe2f8a 100644 --- a/python_modules/dagster/dagster/_core/storage/compute_log_manager.py +++ b/python_modules/dagster/dagster/_core/storage/compute_log_manager.py @@ -26,6 +26,10 @@ class CapturedLogContext( ("external_url", Optional[str]), ("external_stdout_url", Optional[str]), ("external_stderr_url", Optional[str]), + ("stdout_uri_or_path", Optional[str]), + ("stderr_uri_or_path", Optional[str]), + ("stdout_shell_cmd", Optional[str]), + ("stderr_shell_cmd", Optional[str]), ], ) ): @@ -40,6 +44,10 @@ def __new__( external_stdout_url: Optional[str] = None, external_stderr_url: Optional[str] = None, external_url: Optional[str] = None, + stdout_uri_or_path: Optional[str] = None, + stderr_uri_or_path: Optional[str] = None, + stdout_shell_cmd: Optional[str] = None, + stderr_shell_cmd: Optional[str] = None, ): if external_url and (external_stdout_url or external_stderr_url): check.failed( @@ -53,6 +61,10 @@ def __new__( external_stdout_url=external_stdout_url, external_stderr_url=external_stderr_url, external_url=external_url, + stdout_uri_or_path=stdout_uri_or_path, + stderr_uri_or_path=stderr_uri_or_path, + stdout_shell_cmd=stdout_shell_cmd, + stderr_shell_cmd=stderr_shell_cmd, ) @@ -89,6 +101,10 @@ class CapturedLogMetadata( ("stderr_location", Optional[str]), ("stdout_download_url", Optional[str]), ("stderr_download_url", Optional[str]), + ("stdout_uri_or_path", Optional[str]), + ("stderr_uri_or_path", Optional[str]), + ("stdout_shell_cmd", Optional[str]), + ("stderr_shell_cmd", Optional[str]), ], ) ): @@ -102,6 +118,10 @@ def __new__( stderr_location: Optional[str] = None, stdout_download_url: Optional[str] = None, stderr_download_url: Optional[str] = None, + stdout_uri_or_path: Optional[str] = None, + stderr_uri_or_path: Optional[str] = None, + stdout_shell_cmd: Optional[str] = None, + stderr_shell_cmd: Optional[str] = None, ): return super(CapturedLogMetadata, cls).__new__( cls, @@ -109,6 +129,10 @@ def __new__( stderr_location=stderr_location, stdout_download_url=stdout_download_url, stderr_download_url=stderr_download_url, + stdout_download_uri=stdout_uri_or_path, + stderr_download_uri=stderr_uri_or_path, + stdout_shell_cmd=stdout_shell_cmd, + stderr_shell_cmd=stderr_shell_cmd, ) From ce0b20419a3d6f96108818940bb95305ca213946 Mon Sep 17 00:00:00 2001 From: Mathieu Larose Date: Thu, 26 Dec 2024 10:59:55 -0500 Subject: [PATCH 2/2] [dagster-azure] AzureBlobComputeLogManager provides path and shell cmd --- .../dagster_azure/blob/compute_log_manager.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/python_modules/libraries/dagster-azure/dagster_azure/blob/compute_log_manager.py b/python_modules/libraries/dagster-azure/dagster_azure/blob/compute_log_manager.py index 0919956d48f7f..dafb7f4696776 100644 --- a/python_modules/libraries/dagster-azure/dagster_azure/blob/compute_log_manager.py +++ b/python_modules/libraries/dagster-azure/dagster_azure/blob/compute_log_manager.py @@ -237,8 +237,19 @@ def download_url_for_type(self, log_key: Sequence[str], io_type: ComputeIOType): self._download_urls[blob_key] = url return url + def uri_or_path_for_type(self, log_key: Sequence[str], io_type: ComputeIOType): + if not self.is_capture_complete(log_key): + return None + + return self._blob_key(log_key, io_type) + + def shell_command_for_type(self, log_key: Sequence[str], io_type: ComputeIOType): + blob_key = self._blob_key(log_key, io_type) + return f"az storage blob download --account-name {self._storage_account} --container-name {self._container} --name ${blob_key}" + @contextmanager def capture_logs(self, log_key: Sequence[str]) -> Iterator[CapturedLogContext]: + STORAGE_BLOB_DOWNLOAD_SHELL_CMD="az storage blob download --account-name mlarosesandboxlogs --container-name dagster-logs --name" with super().capture_logs(log_key) as local_context: if not self._show_url_only: yield local_context @@ -249,7 +260,13 @@ def capture_logs(self, log_key: Sequence[str]) -> Iterator[CapturedLogContext]: out_url = f"{azure_base_url}/{out_key}" err_url = f"{azure_base_url}/{err_key}" yield CapturedLogContext( - local_context.log_key, external_stdout_url=out_url, external_stderr_url=err_url + local_context.log_key, + external_stdout_url=out_url, + external_stderr_url=err_url, + stdout_uri_or_path=out_key, + stderr_uri_or_path=err_key, + stdout_shell_cmd=self.shell_command_for_type(log_key, ComputeIOType.STDOUT), + stderr_shell_cmd=self.shell_command_for_type(log_key, ComputeIOType.STDERR), ) def _request_user_delegation_key(