From 4a39d233fda53cce176ee0611188b74f0910f7b6 Mon Sep 17 00:00:00 2001 From: Sandy Ryza Date: Fri, 12 Jun 2020 12:15:46 -0700 Subject: [PATCH] config -> config_schema in calls Test Plan: bk Reviewers: schrockn Reviewed By: schrockn Differential Revision: https://dagster.phacility.com/D3476 --- .../dagster_graphql_tests/graphql/setup.py | 14 +++--- .../test_pipelines/test_pipelines/repo.py | 2 +- .../dagster/dagster/config/config_type.py | 4 +- .../dagster/dagster/config/field.py | 2 +- .../dagster/dagster/config/field_utils.py | 2 +- .../core/definitions/decorators/solid.py | 2 +- .../cli_tests/test_config_scaffolder.py | 2 +- .../evaluator_tests/test_composite_descent.py | 27 ++++++----- .../evaluator_tests/test_config_mappings.py | 47 +++++++++++-------- .../config_types_tests/test_config_enums.py | 6 ++- .../config_types_tests/test_config_spec.py | 2 +- .../test_config_type_system.py | 16 +++---- .../config_types_tests/test_type_printer.py | 2 +- .../test_definition_errors.py | 6 ++- .../definitions_tests/test_definitions.py | 4 +- .../engine_tests/test_multiprocessing.py | 2 +- .../test_execution_plan_composite.py | 2 +- .../test_external_step.py | 2 +- .../test_cli_api_running_multiprocessing.py | 2 +- .../snap_tests/test_config_schema_snapshot.py | 2 +- .../snap_tests/test_pipeline_snap.py | 6 ++- .../snap_tests/test_solid_def_snap.py | 2 +- .../system_config_tests/test_system_config.py | 33 ++++++------- .../core_tests/test_naming_collisions.py | 2 +- .../core_tests/test_resource_definition.py | 2 +- .../core_tests/test_solid_aliases.py | 2 +- .../core_tests/test_solid_with_config.py | 4 +- .../py3_tests/test_type_examples_py3.py | 2 +- .../dagster_aws/redshift/resources.py | 4 +- .../emr_tests/test_pyspark.py | 2 +- .../dagster_azure/adls2/system_storage.py | 2 +- .../dagster-celery/dagster_celery/executor.py | 2 +- .../dagster_celery/executor_k8s.py | 2 +- .../dagster-dask/dagster_dask/executor.py | 2 +- .../dagster_databricks/solids.py | 2 +- .../dagster_databricks_tests/test_pyspark.py | 2 +- .../dagster_gcp/bigquery/solids.py | 12 ++--- .../dagster_gcp/dataproc/resources.py | 2 +- .../dagster_gcp/dataproc/solids.py | 2 +- .../dagster_gcp/gcs/system_storage.py | 2 +- .../dagster_github/resources.py | 2 +- .../dagster-shell/dagster_shell/solids.py | 6 +-- .../dagster_shell_tests/test_solids.py | 2 +- .../dagster-ssh/dagster_ssh/solids.py | 2 +- .../libraries/lakehouse/lakehouse/house.py | 2 +- 45 files changed, 135 insertions(+), 116 deletions(-) diff --git a/python_modules/dagster-graphql/dagster_graphql_tests/graphql/setup.py b/python_modules/dagster-graphql/dagster_graphql_tests/graphql/setup.py index 5979cd22369e6..298ec8b574e07 100644 --- a/python_modules/dagster-graphql/dagster_graphql_tests/graphql/setup.py +++ b/python_modules/dagster-graphql/dagster_graphql_tests/graphql/setup.py @@ -253,7 +253,7 @@ def emit_successful_expectation_no_metadata(_context): @pipeline def more_complicated_config(): @solid( - config={ + config_schema={ 'field_one': Field(String), 'field_two': Field(String, is_required=False), 'field_three': Field(String, is_required=False, default_value='some_value'), @@ -272,7 +272,7 @@ def more_complicated_nested_config(): name='a_solid_with_multilayered_config', input_defs=[], output_defs=[], - config={ + config_schema={ 'field_any': Any, 'field_one': String, 'field_two': Field(String, is_required=False), @@ -342,7 +342,7 @@ def hello_world_with_tags(): return solid_that_gets_tags() -@solid(name='solid_with_list', input_defs=[], output_defs=[], config=[int]) +@solid(name='solid_with_list', input_defs=[], output_defs=[], config_schema=[int]) def solid_def(_): return None @@ -406,7 +406,7 @@ def return_any(): @pipeline def pipeline_with_enum_config(): @solid( - config=Enum( + config_schema=Enum( 'TestEnum', [ EnumValue(config_value='ENUM_VALUE_ONE', description='An enum value.'), @@ -677,7 +677,7 @@ def retry_resource_pipeline(): @solid( - config={'fail': bool}, + config_schema={'fail': bool}, input_defs=[InputDefinition('inp', str)], output_defs=[ OutputDefinition(str, 'start_fail', is_required=False), @@ -736,7 +736,7 @@ def return_one(): return 1 @solid( - config={'wait_to_terminate': bool}, + config_schema={'wait_to_terminate': bool}, input_defs=[InputDefinition('one', Int)], output_defs=[OutputDefinition(Int)], ) @@ -747,7 +747,7 @@ def get_input_one(context, one): return one @solid( - config={'wait_to_terminate': bool}, + config_schema={'wait_to_terminate': bool}, input_defs=[InputDefinition('one', Int)], output_defs=[OutputDefinition(Int)], ) diff --git a/python_modules/dagster-test/dagster_test/test_project/test_pipelines/test_pipelines/repo.py b/python_modules/dagster-test/dagster_test/test_project/test_pipelines/test_pipelines/repo.py index ee3c9be1c82ac..09b4880d32eba 100644 --- a/python_modules/dagster-test/dagster_test/test_project/test_pipelines/test_pipelines/repo.py +++ b/python_modules/dagster-test/dagster_test/test_project/test_pipelines/test_pipelines/repo.py @@ -40,7 +40,7 @@ def celery_mode_defs(): ] -@solid(input_defs=[InputDefinition('word', String)], config={'factor': Int}) +@solid(input_defs=[InputDefinition('word', String)], config_schema={'factor': Int}) def multiply_the_word(context, word): return word * context.solid_config['factor'] diff --git a/python_modules/dagster/dagster/config/config_type.py b/python_modules/dagster/dagster/config/config_type.py index fd665cc7440f0..a28dd5eae8b1b 100644 --- a/python_modules/dagster/dagster/config/config_type.py +++ b/python_modules/dagster/dagster/config/config_type.py @@ -196,7 +196,7 @@ class Enum(ConfigType): .. code-block:: python @solid( - config=Field( + config_schema=Field( Enum( 'CowboyType', [ @@ -261,7 +261,7 @@ class Color(enum.Enum): BLUE = enum.auto() @solid( - config={"color": Field(Enum.from_python_enum(Color))} + config_schema={"color": Field(Enum.from_python_enum(Color))} ) def select_color(context): # ... diff --git a/python_modules/dagster/dagster/config/field.py b/python_modules/dagster/dagster/config/field.py index 09157f9d6b14e..aa9c43d26ef5b 100644 --- a/python_modules/dagster/dagster/config/field.py +++ b/python_modules/dagster/dagster/config/field.py @@ -192,7 +192,7 @@ class Field(object): .. code-block::python @solid( - config={ + config_schema={ 'word': Field(str, description='I am a word.'), 'repeats': Field(Int, default_value=1, is_required=False), } diff --git a/python_modules/dagster/dagster/config/field_utils.py b/python_modules/dagster/dagster/config/field_utils.py index 44e5507681fcb..4a6c3ce76b29a 100644 --- a/python_modules/dagster/dagster/config/field_utils.py +++ b/python_modules/dagster/dagster/config/field_utils.py @@ -179,7 +179,7 @@ class Selector(_ConfigHasFields): .. code-block:: python @solid( - config=Field( + config_schema=Field( Selector( { 'haw': {'whom': Field(String, default_value='honua', is_required=False)}, diff --git a/python_modules/dagster/dagster/core/definitions/decorators/solid.py b/python_modules/dagster/dagster/core/definitions/decorators/solid.py index d6d900a5e55b1..f164423b6d79d 100644 --- a/python_modules/dagster/dagster/core/definitions/decorators/solid.py +++ b/python_modules/dagster/dagster/core/definitions/decorators/solid.py @@ -173,7 +173,7 @@ def hello_world(context, foo): return foo @solid( - config={'str_value' : Field(str)} + config_schema={'str_value' : Field(str)} ) def hello_world(context, foo): # context.solid_config is a dictionary with 'str_value' key diff --git a/python_modules/dagster/dagster_tests/cli_tests/test_config_scaffolder.py b/python_modules/dagster/dagster_tests/cli_tests/test_config_scaffolder.py index c055e7d87b95e..7d96c106f7e83 100644 --- a/python_modules/dagster/dagster_tests/cli_tests/test_config_scaffolder.py +++ b/python_modules/dagster/dagster_tests/cli_tests/test_config_scaffolder.py @@ -31,7 +31,7 @@ def test_basic_solids_config(snapshot): name='required_field_solid', input_defs=[], output_defs=[], - config={'required_int': Int}, + config_schema={'required_int': Int}, compute_fn=lambda *_args: fail_me(), ) ], diff --git a/python_modules/dagster/dagster_tests/core_tests/config_types_tests/evaluator_tests/test_composite_descent.py b/python_modules/dagster/dagster_tests/core_tests/config_types_tests/evaluator_tests/test_composite_descent.py index 1dc6b9dff237f..a58ee95a4bce2 100644 --- a/python_modules/dagster/dagster_tests/core_tests/config_types_tests/evaluator_tests/test_composite_descent.py +++ b/python_modules/dagster/dagster_tests/core_tests/config_types_tests/evaluator_tests/test_composite_descent.py @@ -128,7 +128,8 @@ def return_int(context): return context.solid_config @composite_solid( - config={'number': int}, config_fn=lambda cfg: {'return_int': {'config': cfg['number'] + 1}}, + config_schema={'number': int}, + config_fn=lambda cfg: {'return_int': {'config': cfg['number'] + 1}}, ) def layer_three_wrap(): return_int() @@ -140,7 +141,7 @@ def _layer_two_double_wrap_cfg_fn(cfg): return {'layer_three_wrap': {'config': {'number': cfg['number'] + 1}}} @composite_solid( - config={'number': int, 'inject_error': bool}, config_fn=_layer_two_double_wrap_cfg_fn + config_schema={'number': int, 'inject_error': bool}, config_fn=_layer_two_double_wrap_cfg_fn ) def layer_two_double_wrap(): layer_three_wrap() @@ -228,7 +229,7 @@ def add_one(_, num): return num + 1 @composite_solid( - config={}, config_fn=lambda _cfg: {'add_one': {'inputs': {'num': {'value': 2}}}} + config_schema={}, config_fn=lambda _cfg: {'add_one': {'inputs': {'num': {'value': 2}}}} ) def wrap_add_one(): add_one() @@ -250,7 +251,9 @@ def test_double_nested_input_via_config_mapping(): def number(num): return num - @composite_solid(config_fn=lambda _: {'number': {'inputs': {'num': {'value': 4}}}}, config={}) + @composite_solid( + config_fn=lambda _: {'number': {'inputs': {'num': {'value': 4}}}}, config_schema={} + ) def wrap_solid(): # pylint: disable=unused-variable return number() @@ -278,7 +281,7 @@ def wrap_pipeline_double_nested_input(): def test_provide_one_of_two_inputs_via_config(): @solid( - config={'config_field_a': Field(String), 'config_field_b': Field(String)}, + config_schema={'config_field_a': Field(String), 'config_field_b': Field(String)}, input_defs=[InputDefinition('input_a', String), InputDefinition('input_b', String)], ) def basic(context, input_a, input_b): @@ -303,7 +306,7 @@ def basic(context, input_a, input_b): 'inputs': {'input_b': {'value': 'set_input_b'}}, } }, - config={'config_field_a': Field(String), 'config_field_b': Field(String)}, + config_schema={'config_field_a': Field(String), 'config_field_b': Field(String)}, ) def wrap_all_config_one_input(input_a): return basic(input_a) @@ -340,7 +343,7 @@ def required_scalar_config_solid(context): @composite_solid( - config={'override_str': Field(String)}, + config_schema={'override_str': Field(String)}, config_fn=lambda cfg: {'layer2': {'config': cfg['override_str']}}, ) def wrap(): @@ -348,7 +351,7 @@ def wrap(): @composite_solid( - config={'nesting_override': Field(String)}, + config_schema={'nesting_override': Field(String)}, config_fn=lambda cfg: {'layer1': {'config': {'override_str': cfg['nesting_override']}}}, ) def nesting_wrap(): @@ -383,14 +386,14 @@ def get_fully_unwrapped_config(): def test_direct_composite_descent_with_error(): @composite_solid( - config={'override_str': Field(int)}, + config_schema={'override_str': Field(int)}, config_fn=lambda cfg: {'layer2': {'config': cfg['override_str']}}, ) def wrap_coerce_to_wrong_type(): return scalar_config_solid.alias('layer2')() @composite_solid( - config={'nesting_override': Field(int)}, + config_schema={'nesting_override': Field(int)}, config_fn=lambda cfg: {'layer1': {'config': {'override_str': cfg['nesting_override']}}}, ) def nesting_wrap_wrong_type_at_leaf(): @@ -456,7 +459,7 @@ def return_enum(context): return context.solid_config['enum'] @composite_solid( - config={'num': int}, + config_schema={'num': int}, config_fn=lambda cfg: { 'return_enum': {'config': {'enum': 'VALUE_ONE' if cfg['num'] == 1 else 'OTHER'}} }, @@ -489,7 +492,7 @@ def return_int(context): return context.solid_config['num'] @composite_solid( - config={'enum': DagsterEnumType}, + config_schema={'enum': DagsterEnumType}, config_fn=lambda cfg: { 'return_int': {'config': {'num': 1 if cfg['enum'] == TestPythonEnum.VALUE_ONE else 2}} }, diff --git a/python_modules/dagster/dagster_tests/core_tests/config_types_tests/evaluator_tests/test_config_mappings.py b/python_modules/dagster/dagster_tests/core_tests/config_types_tests/evaluator_tests/test_config_mappings.py index 35aa4839a388e..4fdfbe957fb37 100644 --- a/python_modules/dagster/dagster_tests/core_tests/config_types_tests/evaluator_tests/test_config_mappings.py +++ b/python_modules/dagster/dagster_tests/core_tests/config_types_tests/evaluator_tests/test_config_mappings.py @@ -30,7 +30,7 @@ def scalar_config_solid(context): @composite_solid( - config={'override_str': Field(String)}, + config_schema={'override_str': Field(String)}, config_fn=lambda cfg: {'scalar_config_solid': {'config': cfg['override_str']}}, ) def wrap(): @@ -39,7 +39,7 @@ def wrap(): def test_multiple_overrides_pipeline(): @composite_solid( - config={'nesting_override': Field(String)}, + config_schema={'nesting_override': Field(String)}, config_fn=lambda cfg: {'wrap': {'config': {'override_str': cfg['nesting_override']}}}, ) def nesting_wrap(): @@ -132,7 +132,7 @@ def wrap_pipeline(): def test_bad_override(): @composite_solid( - config={'does_not_matter': Field(String)}, + config_schema={'does_not_matter': Field(String)}, config_fn=lambda _cfg: {'scalar_config_solid': {'config': 1234}}, ) def bad_wrap(): @@ -224,7 +224,7 @@ def inner_solid(context): return context.solid_config['inner'] @composite_solid( - config={'override': Int}, + config_schema={'override': Int}, config_fn=lambda cfg: {'inner_solid': {'config': {'inner': str(cfg['override'])}}}, ) def test(): @@ -245,14 +245,14 @@ def inner_solid(context): return context.solid_config['inner'] @composite_solid( - config={'override': Int}, + config_schema={'override': Int}, config_fn=lambda cfg: {'inner_solid': {'config': {'inner': str(cfg['override'])}}}, ) def outer(): return inner_solid() @composite_solid( - config={'override': Int}, + config_schema={'override': Int}, config_fn=lambda cfg: {'outer': {'config': {'override': cfg['override']}}}, ) def test(): @@ -270,7 +270,10 @@ def test_pipeline(): def test_nested_with_inputs(): - @solid(input_defs=[InputDefinition('some_input', String)], config={'basic_key': Field(String)}) + @solid( + input_defs=[InputDefinition('some_input', String)], + config_schema={'basic_key': Field(String)}, + ) def basic(context, some_input): yield Output(context.solid_config['basic_key'] + ' - ' + some_input) @@ -279,7 +282,7 @@ def basic(context, some_input): config_fn=lambda cfg: { 'basic': {'config': {'basic_key': 'override.' + cfg['inner_first']}} }, - config={'inner_first': Field(String)}, + config_schema={'inner_first': Field(String)}, ) def inner_wrap(some_input): return basic(some_input) @@ -292,7 +295,7 @@ def outer_wrap_fn(cfg): } } - @composite_solid(config_fn=outer_wrap_fn, config={'outer_first': Field(String)}) + @composite_solid(config_fn=outer_wrap_fn, config_schema={'outer_first': Field(String)}) def outer_wrap(): return inner_wrap() @@ -310,7 +313,7 @@ def config_mapping_pipeline(): def test_wrap_none_config_and_inputs(): @solid( - config={'config_field_a': Field(String), 'config_field_b': Field(String)}, + config_schema={'config_field_a': Field(String), 'config_field_b': Field(String)}, input_defs=[InputDefinition('input_a', String), InputDefinition('input_b', String)], ) def basic(context, input_a, input_b): @@ -421,7 +424,7 @@ def config_mapping_pipeline(): def test_wrap_all_config_no_inputs(): @solid( - config={'config_field_a': Field(String), 'config_field_b': Field(String)}, + config_schema={'config_field_a': Field(String), 'config_field_b': Field(String)}, input_defs=[InputDefinition('input_a', String), InputDefinition('input_b', String)], ) def basic(context, input_a, input_b): @@ -445,7 +448,7 @@ def basic(context, input_a, input_b): } } }, - config={'config_field_a': Field(String), 'config_field_b': Field(String)}, + config_schema={'config_field_a': Field(String), 'config_field_b': Field(String)}, ) def wrap_all_config_no_inputs(input_a, input_b): return basic(input_a, input_b) @@ -516,7 +519,7 @@ def config_mapping_pipeline(): def test_wrap_all_config_one_input(): @solid( - config={'config_field_a': Field(String), 'config_field_b': Field(String)}, + config_schema={'config_field_a': Field(String), 'config_field_b': Field(String)}, input_defs=[InputDefinition('input_a', String), InputDefinition('input_b', String)], ) def basic(context, input_a, input_b): @@ -541,7 +544,7 @@ def basic(context, input_a, input_b): 'inputs': {'input_b': {'value': 'set_input_b'}}, } }, - config={'config_field_a': Field(String), 'config_field_b': Field(String)}, + config_schema={'config_field_a': Field(String), 'config_field_b': Field(String)}, ) def wrap_all_config_one_input(input_a): return basic(input_a) @@ -606,7 +609,7 @@ def config_mapping_pipeline(): def test_wrap_all_config_and_inputs(): @solid( - config={'config_field_a': Field(String), 'config_field_b': Field(String)}, + config_schema={'config_field_a': Field(String), 'config_field_b': Field(String)}, input_defs=[InputDefinition('input_a', String), InputDefinition('input_b', String)], ) def basic(context, input_a, input_b): @@ -633,7 +636,7 @@ def basic(context, input_a, input_b): }, } }, - config={'config_field_a': Field(String), 'config_field_b': Field(String)}, + config_schema={'config_field_a': Field(String), 'config_field_b': Field(String)}, ) def wrap_all(): return basic() @@ -690,7 +693,9 @@ def config_mapping_pipeline(): def test_empty_config(): # Testing that this definition does *not* raise # See: https://github.com/dagster-io/dagster/issues/1606 - @composite_solid(config_fn=lambda _: {'scalar_config_solid': {'config': 'an input'}}, config={}) + @composite_solid( + config_fn=lambda _: {'scalar_config_solid': {'config': 'an input'}}, config_schema={} + ) def wrap_solid(): # pylint: disable=unused-variable return scalar_config_solid() @@ -706,7 +711,9 @@ def wrap_pipeline(): def test_nested_empty_config(): - @composite_solid(config_fn=lambda _: {'scalar_config_solid': {'config': 'an input'}}, config={}) + @composite_solid( + config_fn=lambda _: {'scalar_config_solid': {'config': 'an input'}}, config_schema={} + ) def wrap_solid(): # pylint: disable=unused-variable return scalar_config_solid() @@ -730,7 +737,9 @@ def test_nested_empty_config_input(): def number(num): return num - @composite_solid(config_fn=lambda _: {'number': {'inputs': {'num': {'value': 4}}}}, config={}) + @composite_solid( + config_fn=lambda _: {'number': {'inputs': {'num': {'value': 4}}}}, config_schema={} + ) def wrap_solid(): # pylint: disable=unused-variable return number() diff --git a/python_modules/dagster/dagster_tests/core_tests/config_types_tests/test_config_enums.py b/python_modules/dagster/dagster_tests/core_tests/config_types_tests/test_config_enums.py index a885b573d285a..68e9c41829854 100644 --- a/python_modules/dagster/dagster_tests/core_tests/config_types_tests/test_config_enums.py +++ b/python_modules/dagster/dagster_tests/core_tests/config_types_tests/test_config_enums.py @@ -42,7 +42,7 @@ def test_enum_in_pipeline_execution(): called = {} @solid( - config={'int_field': Int, 'enum_field': Enum('AnEnum', [EnumValue('ENUM_VALUE')]),} + config_schema={'int_field': Int, 'enum_field': Enum('AnEnum', [EnumValue('ENUM_VALUE')]),} ) def config_me(context): assert context.solid_config['int_field'] == 2 @@ -198,7 +198,9 @@ def test_dict_enum_with_bad_default(): with pytest.raises(DagsterInvalidDefinitionError) as exc_info: @solid( - config={'enum': Field(dagster_enum, is_required=False, default_value=NativeEnum.BAR)} + config_schema={ + 'enum': Field(dagster_enum, is_required=False, default_value=NativeEnum.BAR) + } ) def _enum_bad_dict(_): pass diff --git a/python_modules/dagster/dagster_tests/core_tests/config_types_tests/test_config_spec.py b/python_modules/dagster/dagster_tests/core_tests/config_types_tests/test_config_spec.py index bc41b2f0e5ef3..f70fb19100303 100644 --- a/python_modules/dagster/dagster_tests/core_tests/config_types_tests/test_config_spec.py +++ b/python_modules/dagster/dagster_tests/core_tests/config_types_tests/test_config_spec.py @@ -5,7 +5,7 @@ def test_kitchen_sink(): @solid( - config={ + config_schema={ 'str_field': str, 'int_field': int, 'list_int': [int], diff --git a/python_modules/dagster/dagster_tests/core_tests/config_types_tests/test_config_type_system.py b/python_modules/dagster/dagster_tests/core_tests/config_types_tests/test_config_type_system.py index 127955d24c2f2..29977df975d5d 100644 --- a/python_modules/dagster/dagster_tests/core_tests/config_types_tests/test_config_type_system.py +++ b/python_modules/dagster/dagster_tests/core_tests/config_types_tests/test_config_type_system.py @@ -374,7 +374,7 @@ def addition_composite_solid_config_fn(config): @composite_solid( config_fn=addition_composite_solid_config_fn, - config={ + config_schema={ "a": Field(Int, is_required=False, default_value=1), "b": Field(Int, is_required=False, default_value=2), "c": Int, @@ -401,7 +401,7 @@ def prefix_value(context, v): @composite_solid( config_fn=lambda cfg: {'prefix_value': {'config': {'prefix': cfg['prefix']}}}, - config={'prefix': Field(str, is_required=False, default_value='_id_')}, + config_schema={'prefix': Field(str, is_required=False, default_value='_id_')}, ) def prefix_id(val): return prefix_value(val) @@ -449,7 +449,7 @@ def test_build_optionality(): def test_wrong_solid_name(): - @solid(name='some_solid', input_defs=[], output_defs=[], config=Int) + @solid(name='some_solid', input_defs=[], output_defs=[], config_schema=Int) def some_solid(_): return None @@ -472,7 +472,7 @@ def fail_me(): def dummy_resource(config=None): - return ResourceDefinition(lambda: None, config=config) + return ResourceDefinition(lambda: None, config_schema=config) def test_wrong_resources(): @@ -496,7 +496,7 @@ def test_solid_list_config(): value = [1, 2] called = {} - @solid(name='solid_list_config', input_defs=[], output_defs=[], config=[int]) + @solid(name='solid_list_config', input_defs=[], output_defs=[], config_schema=[int]) def solid_list_config(context): assert context.solid_config == value called['yup'] = True @@ -515,7 +515,7 @@ def pipeline_def(): def test_two_list_types(): @solid( - input_defs=[], config={'list_one': [int], 'list_two': [int]}, + input_defs=[], config_schema={'list_one': [int], 'list_two': [int]}, ) def two_list_type(context): return context.solid_config @@ -526,7 +526,7 @@ def two_list_type(context): ).output_value() == {'list_one': [1], 'list_two': [2]} @solid( - input_defs=[], config={'list_one': [Int], 'list_two': [Int]}, + input_defs=[], config_schema={'list_one': [Int], 'list_two': [Int]}, ) def two_list_type_condensed_syntax(context): return context.solid_config @@ -541,7 +541,7 @@ def two_list_type_condensed_syntax(context): ).output_value() == {'list_one': [1], 'list_two': [2]} @solid( - input_defs=[], config={'list_one': [int], 'list_two': [int]}, + input_defs=[], config_schema={'list_one': [int], 'list_two': [int]}, ) def two_list_type_condensed_syntax_primitives(context): return context.solid_config diff --git a/python_modules/dagster/dagster_tests/core_tests/config_types_tests/test_type_printer.py b/python_modules/dagster/dagster_tests/core_tests/config_types_tests/test_type_printer.py index 7e3d043f0a0a0..5565b954f5fdc 100644 --- a/python_modules/dagster/dagster_tests/core_tests/config_types_tests/test_type_printer.py +++ b/python_modules/dagster/dagster_tests/core_tests/config_types_tests/test_type_printer.py @@ -141,7 +141,7 @@ def test_test_type_pipeline_construction(): def define_solid_for_test_type(name, config): - @solid(name=name, config=config, input_defs=[], output_defs=[]) + @solid(name=name, config_schema=config, input_defs=[], output_defs=[]) def a_solid(_): return None diff --git a/python_modules/dagster/dagster_tests/core_tests/definitions_tests/test_definition_errors.py b/python_modules/dagster/dagster_tests/core_tests/definitions_tests/test_definition_errors.py index 2db4ea00c7f4d..04bfca94456ba 100644 --- a/python_modules/dagster/dagster_tests/core_tests/definitions_tests/test_definition_errors.py +++ b/python_modules/dagster/dagster_tests/core_tests/definitions_tests/test_definition_errors.py @@ -146,7 +146,7 @@ def _a_solid(_context): def test_pass_unrelated_type_to_field_error_resource_definition(): with pytest.raises(DagsterInvalidConfigDefinitionError) as exc_info: - ResourceDefinition(resource_fn=lambda: None, config='wut') + ResourceDefinition(resource_fn=lambda: None, config_schema='wut') assert str(exc_info.value).startswith( "Error defining config. Original value passed: 'wut'. 'wut' cannot be resolved." @@ -155,7 +155,9 @@ def test_pass_unrelated_type_to_field_error_resource_definition(): def test_pass_unrelated_type_in_nested_field_error_resource_definition(): with pytest.raises(DagsterInvalidConfigDefinitionError) as exc_info: - ResourceDefinition(resource_fn=lambda: None, config={'field': {'nested_field': 'wut'}}) + ResourceDefinition( + resource_fn=lambda: None, config_schema={'field': {'nested_field': 'wut'}} + ) assert str(exc_info.value).startswith('Error') assert str(exc_info.value).startswith( diff --git a/python_modules/dagster/dagster_tests/core_tests/definitions_tests/test_definitions.py b/python_modules/dagster/dagster_tests/core_tests/definitions_tests/test_definitions.py index 1275bef42e2bf..efd691e9a3590 100644 --- a/python_modules/dagster/dagster_tests/core_tests/definitions_tests/test_definitions.py +++ b/python_modules/dagster/dagster_tests/core_tests/definitions_tests/test_definitions.py @@ -36,7 +36,7 @@ def produce_string(): @solid( input_defs=[InputDefinition('input_one', String)], output_defs=[OutputDefinition(Any)], - config={'another_field': Int}, + config_schema={'another_field': Int}, ) def solid_one(_context, input_one): raise Exception('should not execute') @@ -113,7 +113,7 @@ def produce_string(): @solid( input_defs=[InputDefinition('input_one', String)], output_defs=[OutputDefinition(Any)], - config={'another_field': Int}, + config_schema={'another_field': Int}, ) def solid_one(_context, input_one): raise Exception('should not execute') diff --git a/python_modules/dagster/dagster_tests/core_tests/engine_tests/test_multiprocessing.py b/python_modules/dagster/dagster_tests/core_tests/engine_tests/test_multiprocessing.py index 45ab941c75a02..c650e0f568107 100644 --- a/python_modules/dagster/dagster_tests/core_tests/engine_tests/test_multiprocessing.py +++ b/python_modules/dagster/dagster_tests/core_tests/engine_tests/test_multiprocessing.py @@ -183,7 +183,7 @@ def waiter(context): return @solid( - input_defs=[InputDefinition('after', Nothing)], config=Field(String), + input_defs=[InputDefinition('after', Nothing)], config_schema=Field(String), ) def writer(context): with open(context.solid_config, 'w') as fd: diff --git a/python_modules/dagster/dagster_tests/core_tests/execution_plan_tests/test_execution_plan_composite.py b/python_modules/dagster/dagster_tests/core_tests/execution_plan_tests/test_execution_plan_composite.py index b2c1d24f8e605..0c74edb36b7fe 100644 --- a/python_modules/dagster/dagster_tests/core_tests/execution_plan_tests/test_execution_plan_composite.py +++ b/python_modules/dagster/dagster_tests/core_tests/execution_plan_tests/test_execution_plan_composite.py @@ -28,7 +28,7 @@ def composite_pipeline(): 'node_a': {'config': {'foo': cfg['foo']}}, 'node_b': {'config': {'bar': cfg['bar']}}, }, - config={'foo': Field(String), 'bar': Int}, + config_schema={'foo': Field(String), 'bar': Int}, ) def composite_with_nested_config_solid_and_config_mapping(): return node_b(node_a()) diff --git a/python_modules/dagster/dagster_tests/core_tests/execution_plan_tests/test_external_step.py b/python_modules/dagster/dagster_tests/core_tests/execution_plan_tests/test_external_step.py index ed20630b09c57..a7620f18e8331 100644 --- a/python_modules/dagster/dagster_tests/core_tests/execution_plan_tests/test_external_step.py +++ b/python_modules/dagster/dagster_tests/core_tests/execution_plan_tests/test_external_step.py @@ -64,7 +64,7 @@ def request_retry_local_external_step_launcher(context): def define_basic_pipeline(): - @solid(required_resource_keys=set(['first_step_launcher']), config={'a': Field(str)}) + @solid(required_resource_keys=set(['first_step_launcher']), config_schema={'a': Field(str)}) def return_two(_): return 2 diff --git a/python_modules/dagster/dagster_tests/core_tests/launcher_tests/test_cli_api_running_multiprocessing.py b/python_modules/dagster/dagster_tests/core_tests/launcher_tests/test_cli_api_running_multiprocessing.py index 5ee093543b53e..d227a7e48f8c1 100644 --- a/python_modules/dagster/dagster_tests/core_tests/launcher_tests/test_cli_api_running_multiprocessing.py +++ b/python_modules/dagster/dagster_tests/core_tests/launcher_tests/test_cli_api_running_multiprocessing.py @@ -122,7 +122,7 @@ def composite_pipeline(): 'node_a': {'config': {'foo': cfg['foo']}}, 'node_b': {'config': {'bar': cfg['bar']}}, }, - config={'foo': Field(String), 'bar': Int}, + config_schema={'foo': Field(String), 'bar': Int}, ) def composite_with_nested_config_solid_and_config_mapping(): return node_b(node_a()) diff --git a/python_modules/dagster/dagster_tests/core_tests/snap_tests/test_config_schema_snapshot.py b/python_modules/dagster/dagster_tests/core_tests/snap_tests/test_config_schema_snapshot.py index 5fc3011b59229..7df9c7e0381c7 100644 --- a/python_modules/dagster/dagster_tests/core_tests/snap_tests/test_config_schema_snapshot.py +++ b/python_modules/dagster/dagster_tests/core_tests/snap_tests/test_config_schema_snapshot.py @@ -229,7 +229,7 @@ def single_solid_pipeline(): def test_kitchen_sink_break_out(): @solid( - config=[ + config_schema=[ { 'opt_list_of_int': Field([int], is_required=False), 'nested_dict': { diff --git a/python_modules/dagster/dagster_tests/core_tests/snap_tests/test_pipeline_snap.py b/python_modules/dagster/dagster_tests/core_tests/snap_tests/test_pipeline_snap.py index 2bc23908cc181..9bea96648478a 100644 --- a/python_modules/dagster/dagster_tests/core_tests/snap_tests/test_pipeline_snap.py +++ b/python_modules/dagster/dagster_tests/core_tests/snap_tests/test_pipeline_snap.py @@ -272,7 +272,9 @@ def noop_pipeline(): def test_deserialize_solid_def_snaps_enum(): @solid( - config=Field(Enum('CowboyType', [EnumValue('good'), EnumValue('bad'), EnumValue('ugly')])) + config_schema=Field( + Enum('CowboyType', [EnumValue('good'), EnumValue('bad'), EnumValue('ugly')]) + ) ) def noop_solid(_): pass @@ -389,7 +391,7 @@ def noop_pipeline(): def test_deserialize_solid_def_snaps_multi_type_config(snapshot): @solid( - config=Field( + config_schema=Field( Permissive( { 'foo': Field(Array(float)), diff --git a/python_modules/dagster/dagster_tests/core_tests/snap_tests/test_solid_def_snap.py b/python_modules/dagster/dagster_tests/core_tests/snap_tests/test_solid_def_snap.py index e431fdabe0193..ad2bf9cbd5e62 100644 --- a/python_modules/dagster/dagster_tests/core_tests/snap_tests/test_solid_def_snap.py +++ b/python_modules/dagster/dagster_tests/core_tests/snap_tests/test_solid_def_snap.py @@ -29,7 +29,7 @@ def test_solid_definition_kitchen_sink(): name='output_two', dagster_type=int, description='desc2', is_required=False ), ], - config={'foo': int}, + config_schema={'foo': int}, description='a description', tags={'a_tag': 'yup'}, required_resource_keys={'b_resource', 'a_resource'}, diff --git a/python_modules/dagster/dagster_tests/core_tests/system_config_tests/test_system_config.py b/python_modules/dagster/dagster_tests/core_tests/system_config_tests/test_system_config.py index e18e176cc0a1e..2e85ba615a93d 100644 --- a/python_modules/dagster/dagster_tests/core_tests/system_config_tests/test_system_config.py +++ b/python_modules/dagster/dagster_tests/core_tests/system_config_tests/test_system_config.py @@ -50,7 +50,7 @@ def test_all_types_provided(): resource_defs={ 'some_resource': ResourceDefinition( lambda: None, - config={ + config_schema={ 'with_default_int': Field(Int, is_required=False, default_value=23434) }, ) @@ -83,7 +83,7 @@ def some_solid(_): resource_defs={ 'some_resource': ResourceDefinition( resource_fn=lambda: None, - config={ + config_schema={ 'with_default_int': Field(Int, is_required=False, default_value=23434) }, ) @@ -143,7 +143,7 @@ def test_solid_dictionary_type(): def define_test_solids_config_pipeline(): @solid( name='int_config_solid', - config=Field(Int, is_required=False), + config_schema=Field(Int, is_required=False), input_defs=[], output_defs=[], ) @@ -152,7 +152,7 @@ def int_config_solid(_): @solid( name='string_config_solid', - config=Field(String, is_required=False), + config_schema=Field(String, is_required=False), input_defs=[], output_defs=[], ) @@ -194,7 +194,7 @@ def test_solid_configs_defaults(): def test_solid_dictionary_some_no_config(): - @solid(name='int_config_solid', config=Int, input_defs=[], output_defs=[]) + @solid(name='int_config_solid', config_schema=Int, input_defs=[], output_defs=[]) def int_config_solid(_): return None @@ -220,14 +220,14 @@ def test_whole_environment(): ModeDefinition( name='test_mode', resource_defs={ - 'test_resource': ResourceDefinition(resource_fn=lambda: None, config=Any) + 'test_resource': ResourceDefinition(resource_fn=lambda: None, config_schema=Any) }, ) ], solid_defs=[ SolidDefinition( name='int_config_solid', - config=Int, + config_schema=Int, input_defs=[], output_defs=[], compute_fn=lambda *args: None, @@ -276,7 +276,7 @@ def _assert_config_none(context, value): solid_defs=[ SolidDefinition( name='int_config_solid', - config=Int, + config_schema=Int, input_defs=[], output_defs=[], compute_fn=lambda context, _inputs: _assert_config_none(context, 234), @@ -302,7 +302,7 @@ def _assert_config_none(context, value): solid_defs=[ SolidDefinition( name='int_config_solid', - config=Field(Int, is_required=False), + config_schema=Field(Int, is_required=False), input_defs=[], output_defs=[], compute_fn=lambda context, _inputs: _assert_config_none(context, 234), @@ -332,7 +332,7 @@ def _assert_config_none(context, value): solid_defs=[ SolidDefinition( name='int_config_solid', - config=Int, + config_schema=Int, input_defs=[], output_defs=[], compute_fn=lambda context, _inputs: _assert_config_none(context, 234), @@ -365,7 +365,7 @@ def test_required_solid_with_required_subfield(): solid_defs=[ SolidDefinition( name='int_config_solid', - config={'required_field': String}, + config_schema={'required_field': String}, input_defs=[], output_defs=[], compute_fn=lambda *_args: None, @@ -404,7 +404,7 @@ def test_optional_solid_with_optional_subfield(): solid_defs=[ SolidDefinition( name='int_config_solid', - config=Field( + config_schema=Field( {'optional_field': Field(String, is_required=False)}, is_required=False ), input_defs=[], @@ -438,7 +438,7 @@ def test_required_resource_with_required_subfield(): ModeDefinition( resource_defs={ 'with_required': ResourceDefinition( - resource_fn=lambda: None, config={'required_field': String}, + resource_fn=lambda: None, config_schema={'required_field': String}, ) } ) @@ -465,7 +465,7 @@ def test_all_optional_field_on_single_resource(): resource_defs={ 'with_optional': ResourceDefinition( resource_fn=lambda: None, - config={'optional_field': Field(String, is_required=False)}, + config_schema={'optional_field': Field(String, is_required=False)}, ) } ) @@ -493,10 +493,11 @@ def test_optional_and_required_context(): name='mixed', resource_defs={ 'optional_resource': ResourceDefinition( - lambda: None, config={'optional_field': Field(String, is_required=False)}, + lambda: None, + config_schema={'optional_field': Field(String, is_required=False)}, ), 'required_resource': ResourceDefinition( - lambda: None, config={'required_field': String}, + lambda: None, config_schema={'required_field': String}, ), }, ) diff --git a/python_modules/dagster/dagster_tests/core_tests/test_naming_collisions.py b/python_modules/dagster/dagster_tests/core_tests/test_naming_collisions.py index 951f299c62e22..d4eae44ff4413 100644 --- a/python_modules/dagster/dagster_tests/core_tests/test_naming_collisions.py +++ b/python_modules/dagster/dagster_tests/core_tests/test_naming_collisions.py @@ -21,7 +21,7 @@ def define_pass_value_solid(name, description=None): description=description, input_defs=[], output_defs=[OutputDefinition(String)], - config={'value': Field(String)}, + config_schema={'value': Field(String)}, ) def pass_value_solid(context): yield Output(context.solid_config['value']) diff --git a/python_modules/dagster/dagster_tests/core_tests/test_resource_definition.py b/python_modules/dagster/dagster_tests/core_tests/test_resource_definition.py index 4f497436db2f2..4dfb0009e83bc 100644 --- a/python_modules/dagster/dagster_tests/core_tests/test_resource_definition.py +++ b/python_modules/dagster/dagster_tests/core_tests/test_resource_definition.py @@ -23,7 +23,7 @@ def define_string_resource(): return ResourceDefinition( - config=String, resource_fn=lambda init_context: init_context.resource_config + config_schema=String, resource_fn=lambda init_context: init_context.resource_config ) diff --git a/python_modules/dagster/dagster_tests/core_tests/test_solid_aliases.py b/python_modules/dagster/dagster_tests/core_tests/test_solid_aliases.py index f988e1eb1a9ad..d0c68a10e03cc 100644 --- a/python_modules/dagster/dagster_tests/core_tests/test_solid_aliases.py +++ b/python_modules/dagster/dagster_tests/core_tests/test_solid_aliases.py @@ -64,7 +64,7 @@ def not_first(prev): def test_aliased_configs(): - @solid(input_defs=[], config=Int) + @solid(input_defs=[], config_schema=Int) def load_constant(context): return context.solid_config diff --git a/python_modules/dagster/dagster_tests/core_tests/test_solid_with_config.py b/python_modules/dagster/dagster_tests/core_tests/test_solid_with_config.py index b6c5d4f1680ef..b827e9c114bfd 100644 --- a/python_modules/dagster/dagster_tests/core_tests/test_solid_with_config.py +++ b/python_modules/dagster/dagster_tests/core_tests/test_solid_with_config.py @@ -10,7 +10,7 @@ def test_basic_solid_with_config(): name='solid_with_context', input_defs=[], output_defs=[], - config={'some_config': Field(String)}, + config_schema={'some_config': Field(String)}, ) def solid_with_context(context): did_get['yep'] = context.solid_config @@ -35,7 +35,7 @@ def _t_fn(*_args): name='solid_with_context', input_defs=[], output_defs=[], - config={'some_config': Field(String)}, + config_schema={'some_config': Field(String)}, ) def solid_with_context(context): raise Exception('should not reach') diff --git a/python_modules/dagster/dagster_tests/py3_tests/test_type_examples_py3.py b/python_modules/dagster/dagster_tests/py3_tests/test_type_examples_py3.py index 873e3e8deaf0f..0d70db8ac6df7 100644 --- a/python_modules/dagster/dagster_tests/py3_tests/test_type_examples_py3.py +++ b/python_modules/dagster/dagster_tests/py3_tests/test_type_examples_py3.py @@ -342,7 +342,7 @@ def hello_world(context) -> str: @solid( - config=Field( + config_schema=Field( Selector( { 'haw': {'whom': Field(String, default_value='honua', is_required=False)}, diff --git a/python_modules/libraries/dagster-aws/dagster_aws/redshift/resources.py b/python_modules/libraries/dagster-aws/dagster_aws/redshift/resources.py index bd797397d7855..3fe4d15952d1d 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws/redshift/resources.py +++ b/python_modules/libraries/dagster-aws/dagster_aws/redshift/resources.py @@ -313,7 +313,7 @@ def define_redshift_config(): @resource( - config=define_redshift_config(), + config_schema=define_redshift_config(), description='Resource for connecting to the Redshift data warehouse', ) def redshift_resource(context): @@ -355,7 +355,7 @@ def example_redshift_solid(context): @resource( - config=define_redshift_config(), + config_schema=define_redshift_config(), description='Fake resource for connecting to the Redshift data warehouse. Usage is identical ' 'to the real redshift_resource. Will always return [(1,)] for the single query case and ' '[[(1,)], [(1,)], [(1,)]] for the multi query case.', diff --git a/python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_pyspark.py b/python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_pyspark.py index 958c8ab709b10..c56bb00904b84 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_pyspark.py +++ b/python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_pyspark.py @@ -50,7 +50,7 @@ def make_df_solid(context): @solid( name='blah', description='this is a test', - config={'foo': str, 'bar': int}, + config_schema={'foo': str, 'bar': int}, input_defs=[InputDefinition('people', DataFrame)], output_defs=[OutputDefinition(DataFrame)], required_resource_keys={'pyspark_step_launcher'}, diff --git a/python_modules/libraries/dagster-azure/dagster_azure/adls2/system_storage.py b/python_modules/libraries/dagster-azure/dagster_azure/adls2/system_storage.py index 84facf824da85..08685f4d3ad41 100644 --- a/python_modules/libraries/dagster-azure/dagster_azure/adls2/system_storage.py +++ b/python_modules/libraries/dagster-azure/dagster_azure/adls2/system_storage.py @@ -9,7 +9,7 @@ @system_storage( name='adls2', is_persistent=True, - config={ + config_schema={ 'adls2_file_system': Field(StringSource, description='ADLS Gen2 file system name'), 'adls2_prefix': Field(StringSource, is_required=False, default_value='dagster'), }, diff --git a/python_modules/libraries/dagster-celery/dagster_celery/executor.py b/python_modules/libraries/dagster-celery/dagster_celery/executor.py index a21f12eb79008..4ad54c2415b70 100644 --- a/python_modules/libraries/dagster-celery/dagster_celery/executor.py +++ b/python_modules/libraries/dagster-celery/dagster_celery/executor.py @@ -32,7 +32,7 @@ } -@executor(name='celery', config=CELERY_CONFIG) +@executor(name='celery', config_schema=CELERY_CONFIG) def celery_executor(init_context): '''Celery-based executor. diff --git a/python_modules/libraries/dagster-celery/dagster_celery/executor_k8s.py b/python_modules/libraries/dagster-celery/dagster_celery/executor_k8s.py index df116acceb9f3..fcfa1126e97cc 100644 --- a/python_modules/libraries/dagster-celery/dagster_celery/executor_k8s.py +++ b/python_modules/libraries/dagster-celery/dagster_celery/executor_k8s.py @@ -53,7 +53,7 @@ def celery_k8s_config(): return cfg -@executor(name=CELERY_K8S_CONFIG_KEY, config=celery_k8s_config()) +@executor(name=CELERY_K8S_CONFIG_KEY, config_schema=celery_k8s_config()) def celery_k8s_job_executor(init_context): '''Celery-based executor which launches tasks as Kubernetes Jobs. diff --git a/python_modules/libraries/dagster-dask/dagster_dask/executor.py b/python_modules/libraries/dagster-dask/dagster_dask/executor.py index b774ab1c4e784..0abe3c9281ea3 100644 --- a/python_modules/libraries/dagster-dask/dagster_dask/executor.py +++ b/python_modules/libraries/dagster-dask/dagster_dask/executor.py @@ -6,7 +6,7 @@ @executor( name='dask', - config={ + config_schema={ 'cluster': Field( Selector( { diff --git a/python_modules/libraries/dagster-databricks/dagster_databricks/solids.py b/python_modules/libraries/dagster-databricks/dagster_databricks/solids.py index ade40fb991173..3aa4734548083 100644 --- a/python_modules/libraries/dagster-databricks/dagster_databricks/solids.py +++ b/python_modules/libraries/dagster-databricks/dagster_databricks/solids.py @@ -62,7 +62,7 @@ def _compute_fn(context, _): input_defs=[InputDefinition(_START, Nothing)], output_defs=[OutputDefinition(Int)], compute_fn=_compute_fn, - config={ + config_schema={ 'run_config': define_databricks_submit_custom_run_config(), 'databricks_host': Field(String, is_required=False), 'databricks_token': Field(String, is_required=True), diff --git a/python_modules/libraries/dagster-databricks/dagster_databricks_tests/test_pyspark.py b/python_modules/libraries/dagster-databricks/dagster_databricks_tests/test_pyspark.py index 8bfe6e9ca1537..10181c06e04e1 100644 --- a/python_modules/libraries/dagster-databricks/dagster_databricks_tests/test_pyspark.py +++ b/python_modules/libraries/dagster-databricks/dagster_databricks_tests/test_pyspark.py @@ -72,7 +72,7 @@ def make_df_solid(context): @solid( name='blah', description='this is a test', - config={'foo': str, 'bar': int}, + config_schema={'foo': str, 'bar': int}, input_defs=[InputDefinition('people', DataFrame)], output_defs=[OutputDefinition(DataFrame)], required_resource_keys={'pyspark_step_launcher'}, diff --git a/python_modules/libraries/dagster-gcp/dagster_gcp/bigquery/solids.py b/python_modules/libraries/dagster-gcp/dagster_gcp/bigquery/solids.py index f0b80ff14f0ec..6fc62d21816e4 100644 --- a/python_modules/libraries/dagster-gcp/dagster_gcp/bigquery/solids.py +++ b/python_modules/libraries/dagster-gcp/dagster_gcp/bigquery/solids.py @@ -50,7 +50,7 @@ def bq_solid_for_queries(sql_queries): name=name, input_defs=[InputDefinition(_START, Nothing)], output_defs=[OutputDefinition(List[DataFrame])], - config=define_bigquery_query_config(), + config_schema=define_bigquery_query_config(), required_resource_keys={'bigquery'}, tags={'kind': 'sql', 'sql': '\n'.join(sql_queries)}, ) @@ -82,7 +82,7 @@ def _solid(context): # pylint: disable=unused-argument @solid( input_defs=[InputDefinition('paths', List[str])], output_defs=[OutputDefinition(Nothing)], - config=BIGQUERY_LOAD_CONFIG, + config_schema=BIGQUERY_LOAD_CONFIG, required_resource_keys={'bigquery'}, ) def import_gcs_paths_to_bq(context, paths): @@ -92,7 +92,7 @@ def import_gcs_paths_to_bq(context, paths): @solid( input_defs=[InputDefinition('df', DataFrame)], output_defs=[OutputDefinition(Nothing)], - config=BIGQUERY_LOAD_CONFIG, + config_schema=BIGQUERY_LOAD_CONFIG, required_resource_keys={'bigquery'}, ) def import_df_to_bq(context, df): @@ -102,7 +102,7 @@ def import_df_to_bq(context, df): @solid( input_defs=[InputDefinition('path', str)], output_defs=[OutputDefinition(Nothing)], - config=BIGQUERY_LOAD_CONFIG, + config_schema=BIGQUERY_LOAD_CONFIG, required_resource_keys={'bigquery'}, ) def import_file_to_bq(context, path): @@ -138,7 +138,7 @@ def _execute_load_in_source(context, source, source_name): @solid( input_defs=[InputDefinition(_START, Nothing)], - config=define_bigquery_create_dataset_config(), + config_schema=define_bigquery_create_dataset_config(), required_resource_keys={'bigquery'}, ) def bq_create_dataset(context): @@ -155,7 +155,7 @@ def bq_create_dataset(context): @solid( input_defs=[InputDefinition(_START, Nothing)], - config=define_bigquery_delete_dataset_config(), + config_schema=define_bigquery_delete_dataset_config(), required_resource_keys={'bigquery'}, ) def bq_delete_dataset(context): diff --git a/python_modules/libraries/dagster-gcp/dagster_gcp/dataproc/resources.py b/python_modules/libraries/dagster-gcp/dagster_gcp/dataproc/resources.py index 663d730ae7113..185c430377343 100644 --- a/python_modules/libraries/dagster-gcp/dagster_gcp/dataproc/resources.py +++ b/python_modules/libraries/dagster-gcp/dagster_gcp/dataproc/resources.py @@ -152,7 +152,7 @@ def cluster_context_manager(self): @resource( - config=define_dataproc_create_cluster_config(), + config_schema=define_dataproc_create_cluster_config(), description='Manage a Dataproc cluster resource', ) def dataproc_resource(context): diff --git a/python_modules/libraries/dagster-gcp/dagster_gcp/dataproc/solids.py b/python_modules/libraries/dagster-gcp/dagster_gcp/dataproc/solids.py index 2697143a8f168..24f898e7a1e9c 100644 --- a/python_modules/libraries/dagster-gcp/dagster_gcp/dataproc/solids.py +++ b/python_modules/libraries/dagster-gcp/dagster_gcp/dataproc/solids.py @@ -6,7 +6,7 @@ @solid( required_resource_keys={'dataproc'}, - config={ + config_schema={ 'job_config': define_dataproc_submit_job_config(), 'job_scoped_cluster': Field( Bool, diff --git a/python_modules/libraries/dagster-gcp/dagster_gcp/gcs/system_storage.py b/python_modules/libraries/dagster-gcp/dagster_gcp/gcs/system_storage.py index 50cbbe841be63..6a71585aea1f7 100644 --- a/python_modules/libraries/dagster-gcp/dagster_gcp/gcs/system_storage.py +++ b/python_modules/libraries/dagster-gcp/dagster_gcp/gcs/system_storage.py @@ -9,7 +9,7 @@ @system_storage( name='gcs', is_persistent=True, - config={ + config_schema={ 'gcs_bucket': Field(StringSource), 'gcs_prefix': Field(StringSource, is_required=False, default_value='dagster'), }, diff --git a/python_modules/libraries/dagster-github/dagster_github/resources.py b/python_modules/libraries/dagster-github/dagster_github/resources.py index 6ce05254caf1b..d5d065d89b0fc 100644 --- a/python_modules/libraries/dagster-github/dagster_github/resources.py +++ b/python_modules/libraries/dagster-github/dagster_github/resources.py @@ -137,7 +137,7 @@ def create_issue(self, repo_name, repo_owner, title, body, installation_id=None) @resource( - config={ + config_schema={ "github_app_id": Field( IntSource, description="Github Application ID, for more info see https://developer.github.com/apps/", diff --git a/python_modules/libraries/dagster-shell/dagster_shell/solids.py b/python_modules/libraries/dagster-shell/dagster_shell/solids.py index 34f1121b76fd9..69aefd36a9f21 100644 --- a/python_modules/libraries/dagster-shell/dagster_shell/solids.py +++ b/python_modules/libraries/dagster-shell/dagster_shell/solids.py @@ -60,7 +60,7 @@ def shell_solid_config(): ), input_defs=[InputDefinition('shell_command', str)], output_defs=[OutputDefinition(str, 'result')], - config=shell_solid_config(), + config_schema=shell_solid_config(), ) def shell_solid(context, shell_command): '''This solid executes a shell command it receives as input. @@ -122,7 +122,7 @@ def create_shell_command_solid( description=description, input_defs=[InputDefinition('start', Nothing)], output_defs=[OutputDefinition(str, 'result')], - config=shell_solid_config(), + config_schema=shell_solid_config(), required_resource_keys=required_resource_keys, tags=tags, ) @@ -190,7 +190,7 @@ def create_shell_script_solid( description=kwargs.pop('description', 'A solid to invoke a shell command.'), input_defs=input_defs or [InputDefinition('start', Nothing)], output_defs=[OutputDefinition(str, 'result')], - config=shell_solid_config(), + config_schema=shell_solid_config(), **kwargs ) def _shell_script_solid(context): diff --git a/python_modules/libraries/dagster-shell/dagster_shell_tests/test_solids.py b/python_modules/libraries/dagster-shell/dagster_shell_tests/test_solids.py index 469536a1af286..6557cb122e565 100644 --- a/python_modules/libraries/dagster-shell/dagster_shell_tests/test_solids.py +++ b/python_modules/libraries/dagster-shell/dagster_shell_tests/test_solids.py @@ -72,7 +72,7 @@ def test_shell_script_solid_no_config_composite(): solid = create_shell_script_solid(os.path.join(script_dir, 'test.sh'), name='foobar') @composite_solid( - config={}, config_fn=lambda cfg: {}, output_defs=[OutputDefinition(str, 'result')] + config_schema={}, config_fn=lambda cfg: {}, output_defs=[OutputDefinition(str, 'result')] ) def composite(): return solid() diff --git a/python_modules/libraries/dagster-ssh/dagster_ssh/solids.py b/python_modules/libraries/dagster-ssh/dagster_ssh/solids.py index 49dfb46dd2fdf..9030320e4b9b1 100644 --- a/python_modules/libraries/dagster-ssh/dagster_ssh/solids.py +++ b/python_modules/libraries/dagster-ssh/dagster_ssh/solids.py @@ -5,7 +5,7 @@ @solid( - config={ + config_schema={ 'local_filepath': Field( str, is_required=True, description='local file path to get or put.' ), diff --git a/python_modules/libraries/lakehouse/lakehouse/house.py b/python_modules/libraries/lakehouse/lakehouse/house.py index e5fc09c519440..4ab3d973bbc68 100644 --- a/python_modules/libraries/lakehouse/lakehouse/house.py +++ b/python_modules/libraries/lakehouse/lakehouse/house.py @@ -163,7 +163,7 @@ def get_computed_asset_solid_def(self, computed_asset, assets_in_pipeline): computed_asset, input_defs, output_def ), output_defs=[output_def], - config=None, + config_schema=None, required_resource_keys=required_resource_keys, positional_inputs=None, )