Merge branch 'main' into feature/3468-databricks_users

.github/workflows/integration-tests.yml

@@ -0,0 +1,61 @@
+name: Integration Tests
+
+on:
+
+  pull_request:
+    types: [opened, synchronize]
+
+  merge_group:
+
+
+jobs:
+  trigger-tests:
+    if: github.event_name == 'pull_request'
+    name: Trigger Tests
+    runs-on: ubuntu-latest
+    environment: "test-trigger-is"
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Generate GitHub App Token
+      id: generate-token
+      uses: actions/create-github-app-token@v1
+      with:
+        app-id: ${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }}
+        private-key: ${{ secrets.DECO_WORKFLOW_TRIGGER_PRIVATE_KEY }}
+        owner: ${{ secrets.ORG_NAME }}
+        repositories: ${{secrets.REPO_NAME}}
+
+    - name: Trigger Workflow in Another Repo
+      env:
+        GH_TOKEN: ${{ steps.generate-token.outputs.token }}
+      run: |
+        gh workflow run terraform-isolated-pr.yml -R ${{ secrets.ORG_NAME }}/${{secrets.REPO_NAME}} \
+        --ref main \
+        -f pull_request_number=${{ github.event.pull_request.number }} \
+        -f commit_sha=${{ github.event.pull_request.head.sha }} 
+
+    
+
+  # Statuses and checks apply to specific commits (by hash). 
+  # Enforcement of required checks is done both at the PR level and the merge queue level.
+  # In case of multiple commits in a single PR, the hash of the squashed commit 
+  # will not match the one for the latest (approved) commit in the PR.
+  # We auto approve the check for the merge queue for two reasons:
+  # * Queue times out due to duration of tests.
+  # * Avoid running integration tests twice, since it was already run at the tip of the branch before squashing.
+  auto-approve:
+    if: github.event_name == 'merge_group'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Mark Check
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        shell: bash
+        run: |
+            gh api -X POST -H "Accept: application/vnd.github+json" \
+              -H "X-GitHub-Api-Version: 2022-11-28" \
+              /repos/${{ github.repository }}/statuses/${{ github.sha }} \
+              -f 'state=success' \
+              -f 'context=Integration Tests Check'

CHANGELOG.md

@@ -1,5 +1,34 @@
 # Version changelog
 
+## [Release] Release v1.55.0
+
+### New Features and Improvements
+
+ * Add `databricks_alert` resource to replace `databricks_sql_alert` ([#4051](https://github.com/databricks/terraform-provider-databricks/pull/4051)).
+ * Add `databricks_query` resource instead of `databricks_sql_query` ([#4103](https://github.com/databricks/terraform-provider-databricks/pull/4103)).
+ * Added resource `databricks_custom_app_integration` ([#4124](https://github.com/databricks/terraform-provider-databricks/pull/4124)).
+ * Handle `schema` attribute in `databricks_pipeline` ([#4137](https://github.com/databricks/terraform-provider-databricks/pull/4137)).
+
+
+### Bug Fixes
+
+ * Change repo used in test ([#4122](https://github.com/databricks/terraform-provider-databricks/pull/4122)).
+
+
+### Documentation
+
+ * Clarify that `graviton` option of `databricks_node_type` could be used on Azure ([#4125](https://github.com/databricks/terraform-provider-databricks/pull/4125)).
+ * Fix argument in example for `databricks_custom_app_integration` ([#4132](https://github.com/databricks/terraform-provider-databricks/pull/4132)).
+ * Fix for UC on AWS guide - use `databricks_aws_unity_catalog_assume_role_policy` where necessary ([#4109](https://github.com/databricks/terraform-provider-databricks/pull/4109)).
+
+
+### Exporter
+
+ * **Breaking change**: Move `databricks_workspace_file` to a separate service ([#4118](https://github.com/databricks/terraform-provider-databricks/pull/4118)).
+ * Exclude some system schemas from export ([#4121](https://github.com/databricks/terraform-provider-databricks/pull/4121)).
+ * Use `List` + iteration instead of call to `ListAll` ([#4123](https://github.com/databricks/terraform-provider-databricks/pull/4123)).
+
+
 ## [Release] Release v1.54.0
 
 ### New Features and Improvements

catalog/resource_system_schema.go

@@ -93,10 +93,21 @@ func ResourceSystemSchema() common.Resource {
 					if err != nil {
 						return err
 					}
+					// only track enabled/legacy schemas
+					if schema.State != catalog.SystemSchemaInfoStateEnableCompleted &&
+						schema.State != catalog.SystemSchemaInfoStateEnableInitialized &&
+						schema.State != catalog.SystemSchemaInfoStateUnavailable {
+						log.Printf("[WARN] %s is not enabled, ignoring it", schemaName)
+						d.SetId("")
+						return nil
+					}
+
 					d.Set("full_name", fmt.Sprintf("system.%s", schemaName))
 					return nil
 				}
 			}
+			log.Printf("[WARN] %s does not exist, ignoring it", schemaName)
+			d.SetId("")
 			return nil
 		},
 		Update: createOrUpdate,

catalog/resource_system_schema_test.go

@@ -239,6 +239,74 @@ func TestSystemSchemaRead_Error(t *testing.T) {
 	assert.Equal(t, "abc|access", d.Id(), "Id should not be empty for error reads")
 }
 
+func TestSystemSchemaRead_NotEnabled(t *testing.T) {
+	d, err := qa.ResourceFixture{
+		Fixtures: []qa.HTTPFixture{
+			{
+				Method:   http.MethodGet,
+				Resource: "/api/2.1/unity-catalog/metastore_summary",
+				Response: catalog.GetMetastoreSummaryResponse{
+					MetastoreId: "abc",
+				},
+			},
+			{
+				Method:   http.MethodGet,
+				Resource: "/api/2.1/unity-catalog/metastores/abc/systemschemas?",
+				Response: catalog.ListSystemSchemasResponse{
+					Schemas: []catalog.SystemSchemaInfo{
+						{
+							Schema: "access",
+							State:  catalog.SystemSchemaInfoStateAvailable,
+						},
+						{
+							Schema: "billing",
+							State:  catalog.SystemSchemaInfoStateEnableCompleted,
+						},
+					},
+				},
+			},
+		},
+		Resource: ResourceSystemSchema(),
+		Read:     true,
+		Removed:  true,
+		ID:       "abc|access",
+	}.Apply(t)
+	assert.NoError(t, err)
+	assert.Equal(t, "", d.Id(), "Id should be empty if a schema is not enabled")
+}
+
+func TestSystemSchemaRead_NotExists(t *testing.T) {
+	d, err := qa.ResourceFixture{
+		Fixtures: []qa.HTTPFixture{
+			{
+				Method:   http.MethodGet,
+				Resource: "/api/2.1/unity-catalog/metastore_summary",
+				Response: catalog.GetMetastoreSummaryResponse{
+					MetastoreId: "abc",
+				},
+			},
+			{
+				Method:   http.MethodGet,
+				Resource: "/api/2.1/unity-catalog/metastores/abc/systemschemas?",
+				Response: catalog.ListSystemSchemasResponse{
+					Schemas: []catalog.SystemSchemaInfo{
+						{
+							Schema: "billing",
+							State:  catalog.SystemSchemaInfoStateEnableCompleted,
+						},
+					},
+				},
+			},
+		},
+		Resource: ResourceSystemSchema(),
+		Read:     true,
+		Removed:  true,
+		ID:       "abc|access",
+	}.Apply(t)
+	assert.NoError(t, err)
+	assert.Equal(t, "", d.Id(), "Id should be empty if a schema does not exist")
+}
+
 func TestSystemSchemaDelete(t *testing.T) {
 	d, err := qa.ResourceFixture{
 		Fixtures: []qa.HTTPFixture{

common/version.go

@@ -3,7 +3,7 @@ package common
 import "context"
 
 var (
-	version = "1.54.0"
+	version = "1.55.0"
 	// ResourceName is resource name without databricks_ prefix
 	ResourceName contextKey = 1
 	// Provider is the current instance of provider

docs/guides/experimental-exporter.md

@@ -111,6 +111,7 @@ Services are just logical groups of resources used for filtering and organizatio
   Please note that for services not marked with **listing**, we'll export resources only if they are referenced from other resources.
 
 * `access` - [databricks_permissions](../resources/permissions.md), [databricks_instance_profile](../resources/instance_profile.md), [databricks_ip_access_list](../resources/ip_access_list.md), [databricks_mws_permission_assignment](../resources/mws_permission_assignment.md) and [databricks_access_control_rule_set](../resources/access_control_rule_set.md).
+* `alerts` - **listing** [databricks_alert](../resources/alert.md).
 * `compute` - **listing** [databricks_cluster](../resources/cluster.md).
 * `dashboards` - **listing** [databricks_dashboard](../resources/dashboard.md).
 * `directories` - **listing** [databricks_directory](../resources/directory.md).  *Please note that directories aren't listed when running in the incremental mode! Only directories with updated notebooks will be emitted.*
@@ -123,13 +124,12 @@ Services are just logical groups of resources used for filtering and organizatio
 * `notebooks` - **listing** [databricks_notebook](../resources/notebook.md).
 * `policies` - **listing** [databricks_cluster_policy](../resources/cluster_policy).
 * `pools` - **listing** [instance pools](../resources/instance_pool.md).
+* `queries` - **listing** [databricks_query](../resources/query.md).
 * `repos` - **listing** [databricks_repo](../resources/repo.md)
 * `secrets` - **listing** [databricks_secret_scope](../resources/secret_scope.md) along with [keys](../resources/secret.md) and [ACLs](../resources/secret_acl.md).
 * `settings` - **listing** [databricks_notification_destination](../resources/notification_destination.md).
-* `sql-alerts` - **listing** [databricks_sql_alert](../resources/sql_alert.md).
-* `sql-dashboards` - **listing** [databricks_sql_dashboard](../resources/sql_dashboard.md) along with associated [databricks_sql_widget](../resources/sql_widget.md) and [databricks_sql_visualization](../resources/sql_visualization.md).
+* `sql-dashboards` - **listing** Legacy [databricks_sql_dashboard](../resources/sql_dashboard.md) along with associated [databricks_sql_widget](../resources/sql_widget.md) and [databricks_sql_visualization](../resources/sql_visualization.md).
 * `sql-endpoints` - **listing** [databricks_sql_endpoint](../resources/sql_endpoint.md) along with [databricks_sql_global_config](../resources/sql_global_config.md).
-* `sql-queries` - **listing** [databricks_sql_query](../resources/sql_query.md).
 * `storage` - only [databricks_dbfs_file](../resources/dbfs_file.md) and [databricks_file](../resources/file.md) referenced in other resources (libraries, init scripts, ...) will be downloaded locally and properly arranged into terraform state.
 * `uc-artifact-allowlist` - **listing** exports [databricks_artifact_allowlist](../resources/artifact_allowlist.md) resources for Unity Catalog Allow Lists attached to the current metastore.
 * `uc-catalogs` - **listing** [databricks_catalog](../resources/catalog.md) and [databricks_workspace_binding](../resources/workspace_binding.md)

docs/resources/alert.md

@@ -13,15 +13,15 @@ resource "databricks_directory" "shared_dir" {
 }
 
 # This will be replaced with new databricks_query resource
-resource "databricks_sql_query" "this" {
-  data_source_id = databricks_sql_endpoint.example.data_source_id
-  name           = "My Query Name"
-  query          = "SELECT 42 as value"
-  parent         = "folders/${databricks_directory.shared_dir.object_id}"
+resource "databricks_query" "this" {
+  warehouse_id = databricks_sql_endpoint.example.id
+  display_name = "My Query Name"
+  query_text   = "SELECT 42 as value"
+  parent_path  = databricks_directory.shared_dir.path
 }
 
 resource "databricks_alert" "alert" {
-  query_id     = databricks_sql_query.this.id
+  query_id     = databricks_query.this.id
   display_name = "TF new alert"
   parent_path  = databricks_directory.shared_dir.path
   condition {
@@ -77,7 +77,11 @@ In addition to all the arguments above, the following attributes are exported:
 
 ## Migrating from `databricks_sql_alert` resource
 
-Under the hood, the new resource uses the same data as the `databricks_sql_alert`, but is exposed via a different API. This means that we can migrate existing alerts without recreating them.  This operation is done in few steps:
+Under the hood, the new resource uses the same data as the `databricks_sql_alert`, but is exposed via a different API. This means that we can migrate existing alerts without recreating them.  
+
+-> It's also recommended to migrate to the `databricks_query` resource - see [databricks_query](query.md) for more details.
+
+This operation is done in few steps:
 
 * Record the ID of existing `databricks_sql_alert`, for example, by executing the `terraform state show databricks_sql_alert.alert` command.
 * Create the code for the new implementation by performing the following changes:
@@ -109,7 +113,7 @@ we'll have a new resource defined as:
 
 ```hcl
 resource "databricks_alert" "alert" {
-  query_id     = databricks_sql_query.this.id
+  query_id     = databricks_query.this.id
   display_name = "My Alert"
   parent_path  = databricks_directory.shared_dir.path
   condition {
@@ -179,6 +183,20 @@ resource "databricks_permissions" "alert_usage" {
 }
 ```
 
+## Access Control
+
+[databricks_permissions](permissions.md#sql-alert-usage) can control which groups or individual users can *Manage*, *Edit*, *Run* or *View* individual alerts.
+
+```hcl
+resource "databricks_permissions" "alert_usage" {
+  sql_alert_id = databricks_alert.alert.id
+  access_control {
+    group_name       = "users"
+    permission_level = "CAN_RUN"
+  }
+}
+```
+
 ## Import
 
 This resource can be imported using alert ID:
@@ -191,6 +209,6 @@ terraform import databricks_alert.this <alert-id>
 
 The following resources are often used in the same context:
 
-* [databricks_sql_query](sql_query.md) to manage Databricks SQL [Queries](https://docs.databricks.com/sql/user/queries/index.html).
-* [databricks_sql_endpoint](sql_endpoint.md) to manage Databricks SQL [Endpoints](https://docs.databricks.com/sql/admin/sql-endpoints.html).
+* [databricks_query](query.md) to manage [Databricks SQL Queries](https://docs.databricks.com/sql/user/queries/index.html).
+* [databricks_sql_endpoint](sql_endpoint.md) to manage [Databricks SQL Endpoints](https://docs.databricks.com/sql/admin/sql-endpoints.html).
 * [databricks_directory](directory.md) to manage directories in [Databricks Workpace](https://docs.databricks.com/workspace/workspace-objects.html).

docs/resources/job.md

@@ -224,14 +224,14 @@ One of the `query`, `dashboard` or `alert` needs to be provided.
 
 * `warehouse_id` - (Required) ID of the (the [databricks_sql_endpoint](sql_endpoint.md)) that will be used to execute the task.  Only Serverless & Pro warehouses are supported right now.
 * `parameters` - (Optional) (Map) parameters to be used for each run of this task. The SQL alert task does not support custom parameters.
-* `query` - (Optional) block consisting of single string field: `query_id` - identifier of the Databricks SQL Query ([databricks_sql_query](sql_query.md)).
+* `query` - (Optional) block consisting of single string field: `query_id` - identifier of the Databricks Query ([databricks_query](query.md)).
 * `dashboard` - (Optional) block consisting of following fields:
   * `dashboard_id` - (Required) (String) identifier of the Databricks SQL Dashboard [databricks_sql_dashboard](sql_dashboard.md).
   * `subscriptions` - (Optional) a list of subscription blocks consisting out of one of the required fields: `user_name` for user emails or `destination_id` - for Alert destination's identifier.
   * `custom_subject` - (Optional) string specifying a custom subject of email sent.
   * `pause_subscriptions` - (Optional) flag that specifies if subscriptions are paused or not.
 * `alert` - (Optional) block consisting of following fields:
-  * `alert_id` - (Required) (String) identifier of the Databricks SQL Alert.
+  * `alert_id` - (Required) (String) identifier of the Databricks Alert ([databricks_alert](alert.md)).
   * `subscriptions` - (Optional) a list of subscription blocks consisting out of one of the required fields: `user_name` for user emails or `destination_id` - for Alert destination's identifier.
   * `pause_subscriptions` - (Optional) flag that specifies if subscriptions are paused or not.
 * `file` - (Optional) block consisting of single string fields:
@@ -372,7 +372,6 @@ This block describes the queue settings of the job:
 * `periodic` - (Optional) configuration block to define a trigger for Periodic Triggers consisting of the following attributes:
   * `interval` - (Required) Specifies the interval at which the job should run. This value is required.
   * `unit` - (Required) Options are {"DAYS", "HOURS", "WEEKS"}.
-
 * `file_arrival` - (Optional) configuration block to define a trigger for [File Arrival events](https://learn.microsoft.com/en-us/azure/databricks/workflows/jobs/file-arrival-triggers) consisting of following attributes:
   * `url` - (Required) URL to be monitored for file arrivals. The path must point to the root or a subpath of the external location. Please note that the URL must have a trailing slash character (`/`).
   * `min_time_between_triggers_seconds` - (Optional) If set, the trigger starts a run only after the specified amount of time passed since the last time the trigger fired. The minimum allowed value is 60 seconds.

docs/resources/pipeline.md

@@ -80,7 +80,8 @@ The following arguments are supported:
 * `photon` - A flag indicating whether to use Photon engine. The default value is `false`.
 * `serverless` - An optional flag indicating if serverless compute should be used for this DLT pipeline.  Requires `catalog` to be set, as it could be used only with Unity Catalog.
 * `catalog` - The name of catalog in Unity Catalog. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `storage`).
-* `target` - The name of a database (in either the Hive metastore or in a UC catalog) for persisting pipeline output data. Configuring the target setting allows you to view and query the pipeline output data from the Databricks UI.
+* `target` - (Optional, String, Conflicts with `schema`) The name of a database (in either the Hive metastore or in a UC catalog) for persisting pipeline output data. Configuring the target setting allows you to view and query the pipeline output data from the Databricks UI.
+* `schema` - (Optional, String, Conflicts with `target`) The default schema (database) where tables are read from or published to. The presence of this attribute implies that the pipeline is in direct publishing mode. 
 * `edition` - optional name of the [product edition](https://docs.databricks.com/data-engineering/delta-live-tables/delta-live-tables-concepts.html#editions). Supported values are: `CORE`, `PRO`, `ADVANCED` (default).  Not required when `serverless` is set to `true`.
 * `channel` - optional name of the release channel for Spark version used by DLT pipeline.  Supported values are: `CURRENT` (default) and `PREVIEW`.
 * `budget_policy_id` - optional string specifying ID of the budget policy for this DLT pipeline.