[ISSUE] Issue with databricks_workspace resource - replaced after global tag change

[lumaron]

Configuration

terraform {
  required_providers {
    azurerm = {
      source = "hashicorp/azurerm"
    }
    databricks = {
      source = "databricks/databricks"
    }
  }
  backend "azurerm" {
<redacted>
  }
}

provider "azurerm" {
  features {}
}

provider "azurerm" {
  features {}
  alias = ""
  subscription_id = ""
}

provider "azurerm" {
  features {}
  alias = ""
  subscription_id = ""
}


provider "databricks" {
  alias      = "account"
  account_id = module.databricks_setup.databricks_account_id
  host       = "https://accounts.azuredatabricks.net"
}

locals {
  host = module.databricks_setup.databricks_workspace_url
}

provider "databricks" {
  alias = "workspace"
  host  = local.host
}

Expected Behavior

All resources updated in-place. Changes applied without issue

Actual Behavior

Databricks workspace resource is replaced (destroyed and created).

Steps to Reproduce

Terraform and provider versions

Initializing provider plugins...
- Finding latest version of hashicorp/azurerm...
- Finding latest version of databricks/databricks...
- Installing hashicorp/azurerm v4.15.0...
- Installed hashicorp/azurerm v4.15.0 (signed by HashiCorp)
- Installing databricks/databricks v1.62.1...
- Installed databricks/databricks v1.62.1 (self-signed, key ID 92A95A66446BCE3F)

Is it a regression?

No idea

Debug Output

Terraform planned the following actions, but then encountered a problem:

  # module.databricks_setup.module.create_databricks_environment.azurerm_databricks_access_connector.access_connector will be updated in-place
  ~ resource "azurerm_databricks_access_connector" "access_connector" {
        id                  = ""
        name                = ""
      ~ tags                = {
            "managed_by"   = "Terraform"
          + "test"         = "testtag"
        }
        # (2 unchanged attributes hidden)

        # (1 unchanged block hidden)
    }

  # module.databricks_setup.module.create_databricks_environment.azurerm_private_endpoint.private_endpoint_bc will be updated in-place
  ~ resource "azurerm_private_endpoint" "private_endpoint_bc" {
        id                            = ""
        name                          = ""
      ~ tags                          = {
            "managed_by"   = "Terraform"
          + "test"         = "testtag"

        }
        # (7 unchanged attributes hidden)

        # (1 unchanged block hidden)
    }

  # module.databricks_setup.module.create_databricks_environment.azurerm_private_endpoint.private_endpoint_dfs will be updated in-place
  ~ resource "azurerm_private_endpoint" "private_endpoint_dfs" {
        id                            = ""
        name                          = ""
      ~ tags                          = {
            "managed_by"   = "Terraform"
          + "test"         = "testtag"
        }
        # (7 unchanged attributes hidden)

        # (1 unchanged block hidden)
    }
  # module.databricks_setup.module.create_databricks_environment.azurerm_private_endpoint.private_endpoint_vc will be updated in-place
  ~ resource "azurerm_private_endpoint" "private_endpoint_vc" {
        id                            = ""
        name                          = ""
      ~ tags                          = {
            "managed_by"   = "Terraform"
          + "test"         = "testtag"
        }
        # (7 unchanged attributes hidden)

        # (1 unchanged block hidden)
    }

  # module.databricks_setup.module.create_databricks_environment.azurerm_resource_group.rg will be updated in-place
  ~ resource "azurerm_resource_group" "rg" {
        id         = ""
        name       = ""
      ~ tags       = {
            "managed_by"   = "Terraform"
          + "test"         = "testtag"
        }
        # (2 unchanged attributes hidden)
    }
  # module.databricks_setup.module.create_databricks_environment.azurerm_storage_account.storage_account will be updated in-place
  ~ resource "azurerm_storage_account" "storage_account" {
        id                                 = ""
        name                               = ""
      ~ tags                               = {
            "managed_by"   = "Terraform"
          + "test"         = "testtag"
        }
        # (96 unchanged attributes hidden)

        # (3 unchanged blocks hidden)
    }

  # module.databricks_setup.module.create_databricks_workspace_managed.module.create_databricks_workspace.azurerm_databricks_workspace.dbw must be replaced
-/+ resource "azurerm_databricks_workspace" "dbw" {
      + disk_encryption_set_id                              = (known after apply)
      ~ id                                                  = "" -> (known after apply)
      - managed_disk_cmk_rotation_to_latest_version_enabled = false -> null
      ~ managed_disk_identity                               = [] -> (known after apply)
      ~ managed_resource_group_id                           = "" -> (known after apply)
        name                                                = ""
      ~ storage_account_identity                            = [
          - {
              - principal_id = ""
              - tenant_id    = ""
              - type         = ""
            },
        ] -> (known after apply)
      ~ tags                                                = {
            "managed_by"   = "Terraform"
          + "test"         = "testtag"
        }
      ~ workspace_id                                        = "" -> (known after apply)
      ~ workspace_url                                       = "" -> (known after apply)
        # (13 unchanged attributes hidden)
      ~ custom_parameters {
          + nat_gateway_name                                     = (known after apply)
          ~ public_ip_name                                       = "" -> (known after apply)
          ~ storage_account_name                                 = "" -> (known after apply)
          ~ storage_account_sku_name                             = "" -> (known after apply)
          ~ virtual_network_id                                   = "" -> (known after apply) # forces replacement
          ~ vnet_address_prefix                                  = "" -> (known after apply)
            # (6 unchanged attributes hidden)
        }
    }
  # module.databricks_setup.module.create_databricks_workspace_managed.module.create_databricks_workspace.module.databricks_networking.azurerm_network_security_group.databricks_network_security_group will be updated in-place
  ~ resource "azurerm_network_security_group" "databricks_network_security_group" {
        id                  = ""
        name                = ""
      ~ tags                = {
            "managed_by"   = "Terraform"
          + "test"         = "testtag"
        }
        # (3 unchanged attributes hidden)
    }
	
	Plan: 1 to add, 7 to change, 1 to destroy.

Changes to Outputs:
  ~ databricks_workspace_id             = "" -> (known after apply)
  ~ databricks_workspace_id_1           = "" -> (known after apply)
  ~ databricks_workspace_url            = "" -> (known after apply)
  ~ managed_resource_group_id           = "" -> (known after apply)
╷
│ Error: cannot read metastore assignment: failed during request visitor: default auth: cannot configure default credentials, please check https://docs.databricks.com/en/dev-tools/auth.html#databricks-client-unified-authentication to configure credentials for your preferred authentication method. Config: azure_client_secret=***, azure_client_id=***, azure_tenant_id=67e4db54-80ae-4739-b54d-5ee94bd6472e. Env: ARM_CLIENT_SECRET, ARM_CLIENT_ID, ARM_TENANT_ID
│ 
│   with module.databricks_setup.module.create_databricks_workspace_managed.module.create_databricks_workspace.databricks_metastore_assignment.enable_unity_catalog,
│   on ../modules/databricks-workspace-uc-enabled/main.tf line 61, in resource "databricks_metastore_assignment" "enable_unity_catalog":
│   61: resource "databricks_metastore_assignment" "enable_unity_catalog" {
│ 

Important Factoids

Parts with empty quotes ("") were redacted.

My project is basically using different modules and submodules to provision 'environment':

• Resource Group
• Key Vault
• Key Vault PE
• Storage Account
• Storage Account PE
• DBAC
• etc

Workspace (it's created after all whole enviornment module is applied) is separate module and consist of submodules, in the end it creates:

• Workspace
• PE for backend
• Secret Scope
• PAT
• admin permissions for account group
• attaches workspace to the metastore

The issue is that when provisioning the infra everything works, but when I want to change global tags (passed as tags=var.tags) workspace resource is trying to be replaced, even though all environment resources are modified in-place.
If I don't modify tags for the environment and just for the workspace it will work - even for the global RG created in the environment module

The fact that Databricks workspace is replaced is messing up passing of databricks.workspace provider for different resources - this is why there's the problem. Basically databricks.workspace is taken from an output which will be changed (to the same value, but well) and it fails

Error: cannot read metastore assignment: failed during request visitor: default auth: cannot configure default credentials

The plan goes through but it fails as soon as the apply begins.

[alexott]

The azurerm_databricks_workspace is managed by Hashicorp, not by Databricks, so open the issue in the https://github.com/hashicorp/terraform-provider-azurerm repository.

The real problem is here:

~ virtual_network_id                                   = "" -> (known after apply) # forces replacement

as you can't change virtual network for the existing workspace.

[lumaron]

Thanks Alex, I will open issue to them as well. It's strange because only tags were changed, so I don't understand why is trying to change it, when module definition looks like it:

#---------------------------------------------------------
# Read Virtual Network
#----------------------------------------------------------

module "data_virtual_network" {
  source             = "../data-virtual-network"
  virtual_network    = var.subscription == "dev" ? "" : ""
  virtual_network_rg = var.subscription == "dev" ? "" : ""
  location           = "westeurope"
}

#---------------------------------------------------------
# Configure Databricks Networking Configuration
#----------------------------------------------------------

module "databricks_networking" {
  source                         = "../databricks-networking"
  location                       = "westeurope"
  nsg_resource_group_name        = var.dbw_resource_group_name
  virtual_network                = module.data_virtual_network.virtual_network_name
  virtual_network_resource_group = module.data_virtual_network.virtual_network_rg
  public_subnet_name             = "adb-public-${var.workspace}-${var.environment}"
  private_subnet_name            = "adb-private-${var.workspace}-${var.environment}"
  public_subnet_address_prefix   = var.public_subnet_address_prefix
  private_subnet_address_prefix  = var.private_subnet_address_prefix
  nsg_name                       = "nsg-${var.workspace}-${var.environment}"
  tags                           = var.tags
  depends_on = [ module.data_virtual_network ]
}

#---------------------------------------------------------
# Create Databricks Workspace
#----------------------------------------------------------

resource "azurerm_databricks_workspace" "dbw" {
  # provider                      = azurerm
  name                                  = "dbw-${var.workspace}-${var.environment}"
  resource_group_name                   = var.dbw_resource_group_name
  sku                                   = "premium"
  location                              = "westeurope"
  managed_resource_group_name           = "rg-${var.workspace}-${var.environment}-managed"
  customer_managed_key_enabled          = true
  public_network_access_enabled         = true
  network_security_group_rules_required = true
  custom_parameters {
    no_public_ip                                         = true
    virtual_network_id                                   = module.data_virtual_network.virtual_network_id
    public_subnet_name                                   = module.databricks_networking.public_subnet
    private_subnet_name                                  = module.databricks_networking.private_subnet
    public_subnet_network_security_group_association_id  = module.databricks_networking.public_association_id
    private_subnet_network_security_group_association_id = module.databricks_networking.private_association_id
  }
  tags = var.tags
  depends_on = [ module.databricks_networking ]
}

[alexott]

it may happen if network information is retrieved using data sources, but it's hard to say without seen the whole code

[lumaron]

Is there any way to change the way it works in such setup? I hardcoded VNET id just for test and now all of the suden I have a cycle coming from different modules, which doesn't make that much sense to me since there are no direct dependencies between them. Honestly I'm just looking for the best balance between modular setup and dynamic provisioning of infra.

Any tips for resolving the issue locally?

Now only permissions and private endpoint is replaced, others are replaced in place:

  # module.databricks_setup.module.create_databricks_workspace_managed.azurerm_private_endpoint.private_endpoint_databricks_dbw must be replaced
-/+ resource "azurerm_private_endpoint" "private_endpoint_databricks_dbw" {
      ~ custom_dns_configs            = [
          - {
              - fqdn         = ""
              - ip_addresses = [
                  - "",
                ]
            },
        ] -> (known after apply)
      ~ id                            = "" -> (known after apply)
        name                          = ""
      ~ network_interface             = [
          - {
              - id   = ""
              - name = ""
            },
        ] -> (known after apply)
      ~ private_dns_zone_configs      = [] -> (known after apply)
      ~ subnet_id                     = "" -> (known after apply) # forces replacement
      ~ tags                          = {
            "managed_by"   = "Terraform"
            "test"         = "testtag"
          + "test2"        = "tag"
        }
        # (3 unchanged attributes hidden)

      ~ private_service_connection {
            name                              = ""
          ~ private_ip_address                = "" -> (known after apply)
            # (5 unchanged attributes hidden)
        }
    }

[...]
  # module.databricks_setup.module.workspace_admins.databricks_permission_assignment.databricks_admin must be replaced
-/+ resource "databricks_permission_assignment" "databricks_admin" {
      ~ id           = "" -> (known after apply)
      ~ principal_id = "" -> (known after apply) # forces replacement
        # (1 unchanged attribute hidden)
    }

  # module.databricks_setup.module.create_databricks_workspace_managed.azurerm_databricks_workspace.dbw will be updated in-place
  ~ resource "azurerm_databricks_workspace" "dbw" {
        id                                                  = ""
        name                                                = ""
      ~ tags                                                = {
            "managed_by"   = "Terraform"
            "test"         = "testtag"
          + "test2"        = "tag"
        }
        # (20 unchanged attributes hidden)

        # (1 unchanged block hidden)
    }

[...]

Plan: 2 to add, 13 to change, 2 to destroy.
╷
│ Error: Cycle: module.databricks_setup.module.create_databricks_environment.azurerm_private_dns_a_record.dns_record_dev_bc (expand), module.databricks_setup.module.create_databricks_environment.azurerm_private_dns_a_record.dns_record_prod_vc (expand), module.databricks_setup.module.create_databricks_environment.output.kv_private_endpoint_name (expand), module.databricks_setup.module.create_databricks_environment.output.storage_account_name (expand), module.databricks_setup.module.create_databricks_environment.azurerm_private_dns_a_record.dns_record_dev_vc (expand), module.databricks_setup.module.create_databricks_environment.output.key_vault_id (expand), module.databricks_setup.module.create_databricks_environment.output.kv_private_endpoint_ip_address (expand), module.databricks_setup.module.create_databricks_environment.azurerm_private_endpoint.private_endpoint_vc, module.databricks_setup.module.create_databricks_environment.azurerm_role_assignment.azroleas (expand), module.databricks_setup.module.create_databricks_environment.azurerm_private_endpoint.private_endpoint_vc (expand), module.databricks_setup.module.create_databricks_environment.azurerm_private_dns_a_record.dns_record_prod_dfs (expand), module.databricks_setup.module.create_databricks_environment.azurerm_private_dns_a_record.dns_record_prod_bc (expand), module.databricks_setup.module.create_databricks_environment.azurerm_private_endpoint.private_endpoint_bc, module.databricks_setup.module.create_databricks_environment.azurerm_private_endpoint.private_endpoint_bc (expand), module.databricks_setup.module.create_databricks_environment.output.access_connector_name (expand), module.databricks_setup.module.create_databricks_environment.azurerm_private_endpoint.private_endpoint_dfs (expand), module.databricks_setup.module.create_databricks_environment.azurerm_private_endpoint.private_endpoint_dfs, module.databricks_setup.module.create_databricks_environment.azurerm_private_dns_a_record.dns_record_dev_dfs (expand), module.databricks_setup.module.create_databricks_environment.output.key_vault_name (expand), module.databricks_setup.module.create_databricks_environment.output.storage_account_rg (expand), module.databricks_setup.module.create_databricks_environment (close), module.databricks_setup.module.create_databricks_environment.azurerm_databricks_access_connector.access_connector, module.databricks_setup.module.create_databricks_environment.azurerm_databricks_access_connector.access_connector (expand), module.databricks_setup.module.create_databricks_environment.output.access_connector_id (expand), module.databricks_setup.module.create_databricks_environment.azurerm_storage_account.storage_account, module.databricks_setup.module.create_databricks_environment.azurerm_storage_account.storage_account (expand), module.databricks_setup.module.create_databricks_environment.output.storage_account_id (expand), module.databricks_setup.module.create_databricks_workspace_managed.azurerm_databricks_workspace.dbw (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.output.private_subnet (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking (close), module.databricks_setup.module.create_databricks_workspace_managed.var.private_subnet_address_prefix (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.var.private_subnet_address_prefix (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.var.private_subnet_name (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.azurerm_subnet.private_subnet (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.azurerm_subnet_network_security_group_association.private_subnet_association (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.output.private_association_id (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.output.network_security_group_name (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.output.network_security_group_id (expand), module.databricks_setup.module.create_databricks_workspace_managed.var.tags (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.var.tags (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.var.nsg_name (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.var.location (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.azurerm_network_security_group.databricks_network_security_group (expand), module.databricks_setup.module.create_databricks_environment.output.dbw_resource_group_name (expand), module.databricks_setup.module.create_databricks_workspace_managed.var.dbw_resource_group_name (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.var.nsg_resource_group_name (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.azurerm_network_security_group.databricks_network_security_group, module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.azurerm_subnet_network_security_group_association.public_subnet_association (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.output.public_association_id (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.var.delegation_actions (expand), module.databricks_setup.module.create_databricks_workspace_managed.var.public_subnet_address_prefix (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.var.public_subnet_address_prefix (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.var.virtual_network (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.var.service_endpoints (expand), module.databricks_setup.module.create_databricks_workspace_managed.var.environment (expand), module.databricks_setup.module.create_databricks_workspace_managed.var.workspace (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.var.public_subnet_name (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.var.service_delegation_name (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.var.delegation_name (expand), module.databricks_setup.module.create_databricks_workspace_managed.var.subscription (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.var.virtual_network_resource_group (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.azurerm_subnet.public_subnet (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking (expand), module.databricks_setup.module.create_databricks_workspace_managed.module.databricks_networking.output.public_subnet (expand), module.databricks_setup.module.create_databricks_workspace_managed.azurerm_databricks_workspace.dbw, module.databricks_setup.module.create_databricks_workspace_managed.output.databricks_workspace_url (expand), module.databricks_setup.output.databricks_workspace_url (expand), local.host (expand), provider["registry.terraform.io/databricks/databricks"].workspace, module.databricks_setup.module.workspace_admins.databricks_permission_assignment.databricks_admin (destroy), module.databricks_setup.module.create_databricks_workspace_managed.azurerm_private_endpoint.private_endpoint_databricks_dbw (destroy), module.databricks_setup.module.create_databricks_environment.azurerm_resource_group.rg, module.databricks_setup.module.create_databricks_environment.azurerm_key_vault.key_vault (expand), module.databricks_setup.module.create_databricks_environment.output.key_vault_uri (expand), module.databricks_setup.module.create_databricks_workspace_managed (expand)
│ 

[lumaron]

• Above happens only when global tag change is in place happens from top module. So change to hardcode is not a change at all. Adding new tag is making the above change.
  When I try to change for example 'public_network_access_enabled' it works right away with no cycling issues from other modules etc, which is insane!