Skip to content

Commit

Permalink
compat-settings alternative
Browse files Browse the repository at this point in the history
  • Loading branch information
alangenfeld committed Feb 23, 2024
1 parent b6cd6e3 commit 0f753bf
Show file tree
Hide file tree
Showing 19 changed files with 84 additions and 13 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,14 @@
import yaml
from dagster._utils import find_free_port, git_repository_root
from dagster._utils.merger import merge_dicts
from dagster_aws_tests.aws_credential_test_utils import get_aws_creds
from dagster_aws.utils import ensure_dagster_aws_tests_import
from dagster_k8s.client import DagsterKubernetesClient

from .integration_utils import IS_BUILDKITE, check_output, get_test_namespace, image_pull_policy

ensure_dagster_aws_tests_import()
from dagster_aws_tests.aws_credential_test_utils import get_aws_creds

TEST_AWS_CONFIGMAP_NAME = "test-aws-env-configmap"
TEST_CONFIGMAP_NAME = "test-env-configmap"
TEST_OTHER_CONFIGMAP_NAME = "test-other-env-configmap"
Expand Down
11 changes: 11 additions & 0 deletions python_modules/dagster-graphql/dagster_graphql/test/utils.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import asyncio
import sys
from contextlib import contextmanager
from pathlib import Path
from typing import Any, Dict, Iterator, Mapping, Optional, Sequence

import dagster._check as check
Expand All @@ -11,6 +13,7 @@
from dagster._core.workspace.load_target import PythonFileTarget
from typing_extensions import Protocol, TypeAlias, TypedDict

from dagster_graphql import __file__ as dagster_graphql_init_py
from dagster_graphql.schema import create_schema


Expand Down Expand Up @@ -222,3 +225,11 @@ def infer_resource_selector(graphql_context: WorkspaceRequestContext, name: str)
selector = infer_repository_selector(graphql_context)
selector = {**selector, **{"resourceName": name}}
return selector


def ensure_dagster_graphql_tests_import() -> None:
dagster_package_root = (Path(dagster_graphql_init_py) / ".." / "..").resolve()
assert (
dagster_package_root / "dagster_graphql_tests"
).exists(), "Could not find dagster_graphql_tests where expected"
sys.path.append(dagster_package_root.as_posix())
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,16 @@
)
from dagster._core.execution.asset_backfill import AssetBackfillData
from dagster._core.instance import DagsterInstance
from dagster._core.test_utils import instance_for_test
from dagster._core.test_utils import ensure_dagster_tests_import, instance_for_test
from dagster_graphql.client.query import LAUNCH_PARTITION_BACKFILL_MUTATION
from dagster_graphql.test.utils import (
GqlResult,
define_out_of_process_context,
execute_dagster_graphql,
main_repo_location_name,
)

ensure_dagster_tests_import()
from dagster_tests.definitions_tests.auto_materialize_tests.scenarios.asset_graphs import (
root_assets_different_partitions_same_downstream,
)
Expand Down
4 changes: 3 additions & 1 deletion python_modules/dagster-test/dagster_test_tests/test_toys.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from dagster._core.definitions.unresolved_asset_job_definition import define_asset_job
from dagster._core.events import DagsterEventType
from dagster._core.storage.fs_io_manager import fs_io_manager
from dagster._core.test_utils import instance_for_test
from dagster._core.test_utils import ensure_dagster_tests_import, instance_for_test
from dagster._utils import file_relative_path
from dagster._utils.temp_file import get_temp_dir
from dagster_test.toys.branches import branch
Expand All @@ -39,6 +39,8 @@
from dagster_test.toys.sensors import get_toys_sensors
from dagster_test.toys.sleepy import sleepy
from dagster_test.toys.software_defined_assets import software_defined_assets

ensure_dagster_tests_import()
from dagster_tests.execution_tests.engine_tests.test_step_delegating_executor import (
test_step_delegating_executor,
)
Expand Down
1 change: 0 additions & 1 deletion python_modules/dagster/dagster/_core/code_pointer.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,6 @@ def load_python_module(
list(remove_from_path_fn()) if remove_from_path_fn else []
) # hook for tests
remove_paths.insert(0, sys.path[0]) # remove the script path

with alter_sys_path(
to_add=([working_directory] if working_directory else []), to_remove=remove_paths
):
Expand Down
10 changes: 10 additions & 0 deletions python_modules/dagster/dagster/_core/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from collections import defaultdict
from concurrent.futures import Future, ThreadPoolExecutor
from contextlib import contextmanager
from pathlib import Path
from signal import Signals
from threading import Event
from typing import (
Expand All @@ -32,6 +33,7 @@
from dagster import (
Permissive,
Shape,
__file__ as dagster_init_py,
_check as check,
fs_io_manager,
)
Expand Down Expand Up @@ -708,3 +710,11 @@ def raise_exception_on_warnings():
if sys.version_info >= (3, 12):
# pendulum sometimes raises DeprecationWarning on python3.12
warnings.filterwarnings("ignore", category=DeprecationWarning, module="pendulum")


def ensure_dagster_tests_import() -> None:
dagster_package_root = (Path(dagster_init_py) / ".." / "..").resolve()
assert (
dagster_package_root / "dagster_tests"
).exists(), "Could not find dagster_tests where expected"
sys.path.append(dagster_package_root.as_posix())
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,15 @@
from dagster._core.execution.retries import RetryState
from dagster._core.execution.stats import RunStepKeyStatsSnapshot
from dagster._core.host_representation import JobHandle
from dagster._core.test_utils import create_run_for_test, environ, instance_for_test
from dagster._core.test_utils import (
create_run_for_test,
ensure_dagster_tests_import,
environ,
instance_for_test,
)
from dagster._serdes import serialize_value

ensure_dagster_tests_import()
from dagster_tests.api_tests.utils import get_bar_repo_handle, get_foo_job_handle


Expand Down
12 changes: 12 additions & 0 deletions python_modules/libraries/dagster-aws/dagster_aws/utils/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import sys
from pathlib import Path
from typing import Optional, TypeVar

import dagster._check as check
Expand All @@ -7,6 +9,8 @@
from packaging import version
from pydantic import Field

from dagster_aws import __file__ as dagster_aws_init_py


def construct_boto_client_retry_config(max_attempts):
check.int_param(max_attempts, "max_attempts")
Expand Down Expand Up @@ -36,3 +40,11 @@ class ResourceWithBoto3Configuration(ConfigurableResource):
profile_name: Optional[str] = Field(
default=None, description="Specifies a profile to connect that session"
)


def ensure_dagster_aws_tests_import() -> None:
dagster_package_root = (Path(dagster_aws_init_py) / ".." / "..").resolve()
assert (
dagster_package_root / "dagster_aws_tests"
).exists(), "Could not find dagster_aws_tests where expected"
sys.path.append(dagster_package_root.as_posix())
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,10 @@
from dagster._core.storage.local_compute_log_manager import IO_TYPE_EXTENSION
from dagster._core.storage.root import LocalArtifactStorage
from dagster._core.storage.runs import SqliteRunStorage
from dagster._core.test_utils import environ, instance_for_test
from dagster._core.test_utils import ensure_dagster_tests_import, environ, instance_for_test
from dagster_aws.s3 import S3ComputeLogManager

ensure_dagster_tests_import()
from dagster_tests.storage_tests.test_captured_log_manager import TestCapturedLogManager

HELLO_WORLD = "Hello World"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@
from dagster._core.storage.event_log import SqliteEventLogStorage
from dagster._core.storage.root import LocalArtifactStorage
from dagster._core.storage.runs import SqliteRunStorage
from dagster._core.test_utils import environ
from dagster._core.test_utils import ensure_dagster_tests_import, environ
from dagster_azure.blob import AzureBlobComputeLogManager, FakeBlobServiceClient

ensure_dagster_tests_import()
from dagster_tests.storage_tests.test_captured_log_manager import TestCapturedLogManager

HELLO_WORLD = "Hello World"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
import pytest
from dagster_aws.utils import ensure_dagster_aws_tests_import

ensure_dagster_aws_tests_import()
from dagster_aws_tests.aws_credential_test_utils import get_aws_creds


Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
import pytest
from dagster_aws.utils import ensure_dagster_aws_tests_import

ensure_dagster_aws_tests_import()
from dagster_aws_tests.aws_credential_test_utils import get_aws_creds


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,13 @@
from dagster._core.storage.event_log import SqliteEventLogStorage
from dagster._core.storage.root import LocalArtifactStorage
from dagster._core.storage.runs import SqliteRunStorage
from dagster._core.test_utils import environ, instance_for_test
from dagster._core.test_utils import ensure_dagster_tests_import, environ, instance_for_test
from dagster_gcp.gcs import GCSComputeLogManager
from dagster_tests.storage_tests.test_captured_log_manager import TestCapturedLogManager
from google.cloud import storage

ensure_dagster_tests_import()
from dagster_tests.storage_tests.test_captured_log_manager import TestCapturedLogManager

HELLO_WORLD = "Hello World"
SEPARATOR = os.linesep if (os.name == "nt" and sys.version_info < (3,)) else "\n"
EXPECTED_LOGS = [
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import pytest
from dagster._core.test_utils import ensure_dagster_tests_import
from dagster_mysql.run_storage import MySQLRunStorage

ensure_dagster_tests_import()
from dagster_tests.storage_tests.utils.daemon_cursor_storage import TestDaemonCursorStorage


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@
import pytest
import yaml
from dagster._core.storage.event_log.base import EventLogCursor
from dagster._core.test_utils import instance_for_test
from dagster._core.test_utils import ensure_dagster_tests_import, instance_for_test
from dagster_mysql.event_log import MySQLEventLogStorage

ensure_dagster_tests_import()
from dagster_tests.storage_tests.utils.event_log_storage import (
TestEventLogStorage,
create_test_event_log_record,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@

import pytest
import yaml
from dagster._core.test_utils import environ, instance_for_test
from dagster._core.test_utils import ensure_dagster_tests_import, environ, instance_for_test
from dagster_mysql.run_storage import MySQLRunStorage

ensure_dagster_tests_import()
from dagster_tests.storage_tests.utils.run_storage import TestRunStorage

TestRunStorage.__test__ = False
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import pytest
from dagster._core.test_utils import ensure_dagster_tests_import
from dagster_postgres.run_storage import PostgresRunStorage

ensure_dagster_tests_import()
from dagster_tests.storage_tests.utils.daemon_cursor_storage import TestDaemonCursorStorage


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@
import pytest
import yaml
from dagster._core.storage.event_log.base import EventLogCursor
from dagster._core.test_utils import instance_for_test
from dagster._core.test_utils import ensure_dagster_tests_import, instance_for_test
from dagster_postgres.event_log import PostgresEventLogStorage

ensure_dagster_tests_import()
from dagster_tests.storage_tests.utils.event_log_storage import (
TestEventLogStorage,
create_test_event_log_record,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import pytest
import yaml
from dagster._core.test_utils import environ, instance_for_test
from dagster._core.test_utils import ensure_dagster_tests_import, environ, instance_for_test
from dagster_postgres.run_storage import PostgresRunStorage

ensure_dagster_tests_import()
from dagster_tests.storage_tests.utils.run_storage import TestRunStorage


Expand Down

0 comments on commit 0f753bf

Please sign in to comment.