From ca1b5ca1c211df104030ee11b59f76aad44279d3 Mon Sep 17 00:00:00 2001 From: Simon Emms Date: Tue, 5 Nov 2024 12:44:02 +0000 Subject: [PATCH] feat(azure): create azure cluster (#428) * chore(devcontainer): set kubeconfig envvar * feat(azure): create azure cluster * feat(azure): retrieve the storage access keys * azure config for vault terraform * fix: lint things * ci: ignore g115 errors in golangci There's no realistic way of avoiding this error in Golang so it doesn't add much value * chore: increase cyclo complexity Necessary to keep the pattern of how the router works. This should be refactored so not required --- .devcontainer/devcontainer.json | 1 + .golangci.yaml | 3 +- docs/docs.go | 32 +++ docs/swagger.json | 32 +++ docs/swagger.yaml | 22 ++ extensions/azure/env.go | 109 ++++++++ extensions/azure/secrets.go | 36 +++ go.mod | 29 ++- go.sum | 40 +++ internal/azure/azure.go | 199 +++++++++++++++ internal/common.go | 2 + internal/controller/argocd.go | 8 +- internal/controller/cluster.go | 17 +- internal/controller/constants.go | 3 + internal/controller/controller.go | 45 +++- internal/controller/domain.go | 10 + internal/controller/git.go | 5 + internal/controller/kubefirst.go | 2 +- internal/controller/repository.go | 22 ++ internal/controller/state.go | 39 +++ internal/controller/tools.go | 13 + internal/controller/users.go | 6 +- internal/controller/vault.go | 18 +- internal/router/api/v1/cluster.go | 33 +++ pkg/constants/cloudProviderDefaults.go | 1 + pkg/providerConfigs/adjustDriver.go | 4 + pkg/providerConfigs/bootstrapSecrets.go | 4 +- pkg/providerConfigs/detokenize.go | 10 + pkg/providerConfigs/types.go | 4 + pkg/types/auth.go | 8 + pkg/types/cloudDefaults.go | 1 + pkg/types/cluster.go | 10 +- pkg/utils/tokens.go | 5 + providers/azure/create.go | 315 ++++++++++++++++++++++++ 34 files changed, 1057 insertions(+), 31 deletions(-) create mode 100644 extensions/azure/env.go create mode 100644 extensions/azure/secrets.go create mode 100644 internal/azure/azure.go create mode 100644 internal/controller/constants.go create mode 100644 providers/azure/create.go diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 3f8b5184..e2b2c708 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -25,6 +25,7 @@ "K1_ACCESS_TOKEN": "feedkray", "K1_LOCAL_DEBUG": "true", "K1_LOCAL_KUBECONFIG_PATH": "/home/vscode/.config/k3d/kubeconfig-dev.yaml", + "KUBECONFIG": "/home/vscode/.config/k3d/kubeconfig-dev.yaml", "KUBEFIRST_TEAM": "true" } } diff --git a/.golangci.yaml b/.golangci.yaml index 8a3d5290..18a704f0 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -103,6 +103,7 @@ linters-settings: - G110 # Decompression bombs: we can check these manually when submitting code - G306 # Poor file permissions used when creating a directory: we can check these manually when submitting code - G404 # Use of weak random number generator (math/rand instead of crypto/rand): we can live with these + - G115 # No realistic way of avoiding this in Go when converting from int to uint stylecheck: checks: @@ -110,7 +111,7 @@ linters-settings: - "-ST1003" # this is covered by a different linter gocyclo: - min-complexity: 60 + min-complexity: 68 exhaustive: check-generated: false diff --git a/docs/docs.go b/docs/docs.go index 56be843f..df7731d4 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -1013,6 +1013,23 @@ const docTemplate = `{ } } }, + "types.AzureAuth": { + "type": "object", + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + }, + "subscription_id": { + "type": "string" + }, + "tenant_id": { + "type": "string" + } + } + }, "types.CivoAuth": { "type": "object", "properties": { @@ -1114,6 +1131,13 @@ const docTemplate = `{ "aws_kms_key_id": { "type": "string" }, + "azure_auth": { + "$ref": "#/definitions/types.AzureAuth" + }, + "azure_dns_zone_resource_group": { + "description": "Azure", + "type": "string" + }, "civo_auth": { "$ref": "#/definitions/types.CivoAuth" }, @@ -1309,6 +1333,13 @@ const docTemplate = `{ "aws_auth": { "$ref": "#/definitions/types.AWSAuth" }, + "azure_auth": { + "$ref": "#/definitions/types.AzureAuth" + }, + "azure_dns_zone_resource_group": { + "description": "Azure", + "type": "string" + }, "civo_auth": { "$ref": "#/definitions/types.CivoAuth" }, @@ -1317,6 +1348,7 @@ const docTemplate = `{ "enum": [ "akamai", "aws", + "azure", "civo", "digitalocean", "google", diff --git a/docs/swagger.json b/docs/swagger.json index bed340d2..e7ac7d43 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -1007,6 +1007,23 @@ } } }, + "types.AzureAuth": { + "type": "object", + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + }, + "subscription_id": { + "type": "string" + }, + "tenant_id": { + "type": "string" + } + } + }, "types.CivoAuth": { "type": "object", "properties": { @@ -1108,6 +1125,13 @@ "aws_kms_key_id": { "type": "string" }, + "azure_auth": { + "$ref": "#/definitions/types.AzureAuth" + }, + "azure_dns_zone_resource_group": { + "description": "Azure", + "type": "string" + }, "civo_auth": { "$ref": "#/definitions/types.CivoAuth" }, @@ -1303,6 +1327,13 @@ "aws_auth": { "$ref": "#/definitions/types.AWSAuth" }, + "azure_auth": { + "$ref": "#/definitions/types.AzureAuth" + }, + "azure_dns_zone_resource_group": { + "description": "Azure", + "type": "string" + }, "civo_auth": { "$ref": "#/definitions/types.CivoAuth" }, @@ -1311,6 +1342,7 @@ "enum": [ "akamai", "aws", + "azure", "civo", "digitalocean", "google", diff --git a/docs/swagger.yaml b/docs/swagger.yaml index b914fe70..f61e9a55 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -19,6 +19,17 @@ definitions: token: type: string type: object + types.AzureAuth: + properties: + client_id: + type: string + client_secret: + type: string + subscription_id: + type: string + tenant_id: + type: string + type: object types.CivoAuth: properties: token: @@ -82,6 +93,11 @@ definitions: type: boolean aws_kms_key_id: type: string + azure_auth: + $ref: '#/definitions/types.AzureAuth' + azure_dns_zone_resource_group: + description: Azure + type: string civo_auth: $ref: '#/definitions/types.CivoAuth' cloud_provider: @@ -207,12 +223,18 @@ definitions: description: Auth aws_auth: $ref: '#/definitions/types.AWSAuth' + azure_auth: + $ref: '#/definitions/types.AzureAuth' + azure_dns_zone_resource_group: + description: Azure + type: string civo_auth: $ref: '#/definitions/types.CivoAuth' cloud_provider: enum: - akamai - aws + - azure - civo - digitalocean - google diff --git a/extensions/azure/env.go b/extensions/azure/env.go new file mode 100644 index 00000000..6a6f6b2c --- /dev/null +++ b/extensions/azure/env.go @@ -0,0 +1,109 @@ +package azure + +import ( + "fmt" + "log" + "strconv" + "strings" + + "github.com/konstructio/kubefirst-api/internal/vault" + "github.com/konstructio/kubefirst-api/pkg/k8s" + "github.com/konstructio/kubefirst-api/pkg/providerConfigs" + pkgtypes "github.com/konstructio/kubefirst-api/pkg/types" + "k8s.io/client-go/kubernetes" +) + +func readVaultTokenFromSecret(clientset kubernetes.Interface) string { + existingKubernetesSecret, err := k8s.ReadSecretV2(clientset, vault.VaultNamespace, vault.VaultSecretName) + if err != nil || existingKubernetesSecret == nil { + log.Printf("Error reading existing Secret data: %s", err) + return "" + } + + return existingKubernetesSecret["root-token"] +} + +func GetAzureTerraformEnvs(envs map[string]string, cl *pkgtypes.Cluster) map[string]string { + envs["ARM_CLIENT_ID"] = cl.AzureAuth.ClientID + envs["ARM_CLIENT_SECRET"] = cl.AzureAuth.ClientSecret + envs["ARM_TENANT_ID"] = cl.AzureAuth.TenantID + envs["ARM_SUBSCRIPTION_ID"] = cl.AzureAuth.SubscriptionID + envs["TF_VAR_arm_client_id"] = cl.AzureAuth.ClientID + envs["TF_VAR_arm_client_secret"] = cl.AzureAuth.ClientSecret + envs["TF_VAR_arm_tenant_id"] = cl.AzureAuth.TenantID + envs["TF_VAR_arm_subscription_id"] = cl.AzureAuth.SubscriptionID + envs["TF_VAR_azure_storage_account"] = cl.StateStoreCredentials.Name + envs["TF_VAR_azure_storage_access_key"] = cl.StateStoreCredentials.SecretAccessKey + + return envs +} + +func GetGithubTerraformEnvs(envs map[string]string, cl *pkgtypes.Cluster) map[string]string { + envs["GITHUB_TOKEN"] = cl.GitAuth.Token + envs["GITHUB_OWNER"] = cl.GitAuth.Owner + envs["TF_VAR_atlantis_repo_webhook_secret"] = cl.AtlantisWebhookSecret + envs["TF_VAR_kbot_ssh_public_key"] = cl.GitAuth.PublicKey + envs["ARM_CLIENT_ID"] = cl.AzureAuth.ClientID + envs["ARM_CLIENT_SECRET"] = cl.AzureAuth.ClientSecret + envs["ARM_TENANT_ID"] = cl.AzureAuth.TenantID + envs["ARM_SUBSCRIPTION_ID"] = cl.AzureAuth.SubscriptionID + + return envs +} + +func GetGitlabTerraformEnvs(envs map[string]string, gid int, cl *pkgtypes.Cluster) map[string]string { + envs["GITLAB_TOKEN"] = cl.GitAuth.Token + envs["GITLAB_OWNER"] = cl.GitAuth.Owner + envs["TF_VAR_atlantis_repo_webhook_secret"] = cl.AtlantisWebhookSecret + envs["TF_VAR_atlantis_repo_webhook_url"] = cl.AtlantisWebhookURL + envs["TF_VAR_kbot_ssh_public_key"] = cl.GitAuth.PublicKey + envs["ARM_CLIENT_ID"] = cl.AzureAuth.ClientID + envs["ARM_CLIENT_SECRET"] = cl.AzureAuth.ClientSecret + envs["ARM_TENANT_ID"] = cl.AzureAuth.TenantID + envs["ARM_SUBSCRIPTION_ID"] = cl.AzureAuth.SubscriptionID + envs["TF_VAR_owner_group_id"] = strconv.Itoa(gid) + envs["TF_VAR_gitlab_owner"] = cl.GitAuth.Owner + + return envs +} + +func GetUsersTerraformEnvs(clientset kubernetes.Interface, cl *pkgtypes.Cluster, envs map[string]string) map[string]string { + envs["VAULT_TOKEN"] = readVaultTokenFromSecret(clientset) + envs["VAULT_ADDR"] = providerConfigs.VaultPortForwardURL + envs[fmt.Sprintf("%s_TOKEN", strings.ToUpper(cl.GitProvider))] = cl.GitAuth.Token + envs[fmt.Sprintf("%s_OWNER", strings.ToUpper(cl.GitProvider))] = cl.GitAuth.Owner + envs["ARM_CLIENT_ID"] = cl.AzureAuth.ClientID + envs["ARM_CLIENT_SECRET"] = cl.AzureAuth.ClientSecret + envs["ARM_TENANT_ID"] = cl.AzureAuth.TenantID + envs["ARM_SUBSCRIPTION_ID"] = cl.AzureAuth.SubscriptionID + + return envs +} + +func GetVaultTerraformEnvs(clientset kubernetes.Interface, cl *pkgtypes.Cluster, envs map[string]string) map[string]string { + envs[fmt.Sprintf("%s_TOKEN", strings.ToUpper(cl.GitProvider))] = cl.GitAuth.Token + envs[fmt.Sprintf("%s_OWNER", strings.ToUpper(cl.GitProvider))] = cl.GitAuth.Owner + envs["TF_VAR_email_address"] = cl.AlertsEmail + envs["TF_VAR_vault_addr"] = providerConfigs.VaultPortForwardURL + envs["TF_VAR_vault_token"] = readVaultTokenFromSecret(clientset) + envs[fmt.Sprintf("TF_VAR_%s_token", cl.GitProvider)] = cl.GitAuth.Token + envs["VAULT_ADDR"] = providerConfigs.VaultPortForwardURL + envs["VAULT_TOKEN"] = readVaultTokenFromSecret(clientset) + envs["TF_VAR_civo_token"] = cl.CivoAuth.Token + envs["TF_VAR_atlantis_repo_webhook_secret"] = cl.AtlantisWebhookSecret + envs["TF_VAR_atlantis_repo_webhook_url"] = cl.AtlantisWebhookURL + envs["TF_VAR_kbot_ssh_private_key"] = cl.GitAuth.PrivateKey + envs["TF_VAR_kbot_ssh_public_key"] = cl.GitAuth.PublicKey + envs["TF_VAR_cloudflare_origin_ca_api_key"] = cl.CloudflareAuth.OriginCaIssuerKey + envs["TF_VAR_cloudflare_api_key"] = cl.CloudflareAuth.APIToken + envs["ARM_CLIENT_ID"] = cl.AzureAuth.ClientID + envs["ARM_CLIENT_SECRET"] = cl.AzureAuth.ClientSecret + envs["ARM_TENANT_ID"] = cl.AzureAuth.TenantID + envs["ARM_SUBSCRIPTION_ID"] = cl.AzureAuth.SubscriptionID + + if cl.GitProvider == "gitlab" { + envs["TF_VAR_owner_group_id"] = fmt.Sprint(cl.GitlabOwnerGroupID) + } + + return envs +} diff --git a/extensions/azure/secrets.go b/extensions/azure/secrets.go new file mode 100644 index 00000000..c06cf7fb --- /dev/null +++ b/extensions/azure/secrets.go @@ -0,0 +1,36 @@ +package azure + +import ( + "fmt" + + providerConfig "github.com/konstructio/kubefirst-api/pkg/providerConfigs" + pkgtypes "github.com/konstructio/kubefirst-api/pkg/types" + "github.com/rs/zerolog/log" + "k8s.io/client-go/kubernetes" +) + +func BootstrapAzureMgmtCluster(clientset kubernetes.Interface, cl *pkgtypes.Cluster, destinationGitopsRepoURL string) error { + opts := providerConfig.BootstrapOptions{ + GitUser: cl.GitAuth.User, + DestinationGitopsRepoURL: destinationGitopsRepoURL, + GitProtocol: cl.GitProtocol, + CloudflareAPIToken: cl.CloudflareAuth.APIToken, + DNSProvider: cl.DNSProvider, + CloudProvider: cl.CloudProvider, + HTTPSPassword: cl.GitAuth.Token, + SSHToken: cl.GitAuth.PrivateKey, + } + + if err := providerConfig.BootstrapMgmtCluster(clientset, opts); err != nil { + log.Error().Msgf("unable to bootstrap management cluster: %s", err) + return fmt.Errorf("unable to bootstrap management cluster: %w", err) + } + + // Create secrets + if err := providerConfig.BootstrapSecrets(clientset, cl); err != nil { + log.Error().Msgf("unable to bootstrap secrets: %s", err) + return fmt.Errorf("unable to bootstrap secrets: %w", err) + } + + return nil +} diff --git a/go.mod b/go.mod index ad65c2e2..1ed729c5 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,12 @@ require ( cloud.google.com/go/container v1.24.0 cloud.google.com/go/secretmanager v1.10.0 cloud.google.com/go/storage v1.29.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dns/armdns v1.2.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1 github.com/argoproj/argo-cd/v2 v2.6.7 github.com/argoproj/gitops-engine v0.7.3 github.com/atotto/clipboard v0.1.4 @@ -44,7 +50,7 @@ require ( github.com/thanhpk/randstr v1.0.6 go.mongodb.org/mongo-driver v1.10.3 golang.org/x/oauth2 v0.8.0 - golang.org/x/text v0.14.0 + golang.org/x/text v0.16.0 google.golang.org/api v0.126.0 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.27.1 @@ -55,13 +61,18 @@ require ( require ( dario.cat/mergo v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/go-resty/resty/v2 v2.11.0 // indirect github.com/go-test/deep v1.0.4 // indirect + github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/hashicorp/go-hclog v1.3.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gotest.tools/v3 v3.4.0 // indirect ) @@ -158,7 +169,7 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/s2a-go v0.1.4 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect github.com/googleapis/gax-go/v2 v2.11.0 // indirect github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect @@ -246,15 +257,15 @@ require ( github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect go.opencensus.io v0.24.0 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect - golang.org/x/crypto v0.21.0 + golang.org/x/crypto v0.25.0 golang.org/x/exp v0.0.0-20231006140011-7918f672742d - golang.org/x/mod v0.13.0 - golang.org/x/net v0.22.0 // indirect - golang.org/x/sync v0.4.0 - golang.org/x/sys v0.18.0 // indirect - golang.org/x/term v0.18.0 + golang.org/x/mod v0.17.0 + golang.org/x/net v0.27.0 // indirect + golang.org/x/sync v0.7.0 + golang.org/x/sys v0.22.0 // indirect + golang.org/x/term v0.22.0 golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.14.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect diff --git a/go.sum b/go.sum index 362d37d2..75f8d73d 100644 --- a/go.sum +++ b/go.sum @@ -59,7 +59,22 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-sdk-for-go v55.0.0+incompatible h1:L4/vUGbg1Xkw5L20LZD+hJI5I+ibWSytqQ68lTCfLwY= github.com/Azure/azure-sdk-for-go v55.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dns/armdns v1.2.0 h1:lpOxwrQ919lCZoNCd69rVt8u1eLZuMORrGXqy8sNf3c= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dns/armdns v1.2.0/go.mod h1:fSvRkb8d26z9dbL40Uf/OO6Vo9iExtZK3D0ulRV+8M0= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1 h1:cf+OIKbkmMHBaC3u78AXomweqM0oxQSgBXRZf3WH4yM= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1/go.mod h1:ap1dmS6vQKJxSMNiGJcq4QuUQkOynyD93gLw6MDF7ek= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -71,6 +86,8 @@ github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcP github.com/Azure/go-autorest/autorest/validation v0.1.0/go.mod h1:Ha3z/SqBeaalWQvokg3NZAlQTalVMtOIAs1aGK7G6u8= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/GoogleCloudPlatform/k8s-cloud-provider v1.16.1-0.20210702024009-ea6160c1d0e3/go.mod h1:8XasY4ymP2V/tn2OOV9ZadmiTE1FIB/h3W+yNlPttKw= @@ -509,6 +526,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= @@ -600,6 +619,8 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -755,6 +776,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kubefirst/metrics-client v0.3.0 h1:zCug82pEzeWhHhpeYQvdhytRNDxrLxX18dPQ5PSxY3s= github.com/kubefirst/metrics-client v0.3.0/go.mod h1:GR7wsMcyYhd+EU67PeuMCBYFE6OJ7P/j5OI5BLOoRMc= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= @@ -975,6 +997,8 @@ github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -1287,6 +1311,8 @@ golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1344,6 +1370,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1414,6 +1442,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1446,6 +1476,8 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1558,6 +1590,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1570,6 +1604,8 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1589,6 +1625,8 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1676,6 +1714,8 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/azure/azure.go b/internal/azure/azure.go new file mode 100644 index 00000000..29ee8a57 --- /dev/null +++ b/internal/azure/azure.go @@ -0,0 +1,199 @@ +package azure + +import ( + "context" + "fmt" + "os" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dns/armdns" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" +) + +var defaultTags = map[string]*string{ + "ProvisionedBy": to.Ptr("kubefirst"), +} + +type Keys struct { + Key1 string + Key2 string +} + +type Client struct { + cred *azidentity.DefaultAzureCredential + subscriptionID string +} + +func (c *Client) newDNSClientFactory() (*armdns.ClientFactory, error) { + client, err := armdns.NewClientFactory(c.subscriptionID, c.cred, nil) + if err != nil { + return nil, fmt.Errorf("failed to create armdns client: %w", err) + } + return client, nil +} + +func (c *Client) newResourceClientFactory() (*armresources.ClientFactory, error) { + client, err := armresources.NewClientFactory(c.subscriptionID, c.cred, nil) + if err != nil { + return nil, fmt.Errorf("failed to create armresources client: %w", err) + } + return client, nil +} + +func (c *Client) newStorageClientFactory() (*armstorage.ClientFactory, error) { + client, err := armstorage.NewClientFactory(c.subscriptionID, c.cred, nil) + if err != nil { + return nil, fmt.Errorf("failed to create armstorage client: %w", err) + } + return client, nil +} + +func (c *Client) CreateBlobContainer(ctx context.Context, storageAccountName, containerName string) (*azblob.CreateContainerResponse, error) { + client, err := azblob.NewClient(fmt.Sprintf("https://%s.blob.core.windows.net", storageAccountName), c.cred, nil) + if err != nil { + return nil, fmt.Errorf("failed to create azblob client: %w", err) + } + + resp, err := client.CreateContainer(ctx, containerName, &azblob.CreateContainerOptions{ + Metadata: defaultTags, + }) + if err != nil { + return nil, fmt.Errorf("failed to create container: %w", err) + } + + return &resp, nil +} + +func (c *Client) CreateResourceGroup(ctx context.Context, name, location string) (*armresources.ResourceGroup, error) { + client, err := c.newResourceClientFactory() + if err != nil { + return nil, err + } + + parameters := armresources.ResourceGroup{ + Location: to.Ptr(location), + Tags: defaultTags, + } + + resp, err := client.NewResourceGroupsClient().CreateOrUpdate(ctx, name, parameters, nil) + if err != nil { + return nil, fmt.Errorf("failed to create azure resource group: %w", err) + } + + return &resp.ResourceGroup, nil +} + +func (c *Client) CreateStorageAccount(ctx context.Context, location, resourceGroup, storageAccountName string) (*armstorage.Account, error) { + client, err := c.newStorageClientFactory() + if err != nil { + return nil, err + } + + params := armstorage.AccountCreateParameters{ + Kind: to.Ptr(armstorage.KindStorageV2), + Location: to.Ptr(location), + SKU: &armstorage.SKU{ + Name: to.Ptr(armstorage.SKUNameStandardGRS), + }, + Properties: &armstorage.AccountPropertiesCreateParameters{ + AccessTier: to.Ptr(armstorage.AccessTierCool), + AllowBlobPublicAccess: to.Ptr(false), + Encryption: &armstorage.Encryption{ + KeySource: to.Ptr(armstorage.KeySourceMicrosoftStorage), + Services: &armstorage.EncryptionServices{ + // We're only using blob storage here, so the other types aren't set + Blob: &armstorage.EncryptionService{ + Enabled: to.Ptr(true), + KeyType: to.Ptr(armstorage.KeyTypeAccount), + }, + }, + }, + MinimumTLSVersion: to.Ptr(armstorage.MinimumTLSVersionTLS12), + }, + Tags: defaultTags, + } + + poller, err := client.NewAccountsClient().BeginCreate( + ctx, + resourceGroup, + storageAccountName, + params, + nil, + ) + if err != nil { + return nil, fmt.Errorf("storage account creation request failed: %w", err) + } + + resp, err := poller.PollUntilDone(ctx, nil) + if err != nil { + return nil, fmt.Errorf("failed to create storage account: %w", err) + } + + return &resp.Account, nil +} + +func (c *Client) GetStorageAccessKeys(ctx context.Context, resourceGroup, storageAccountName string) (*Keys, error) { + client, err := c.newStorageClientFactory() + if err != nil { + return nil, err + } + + keys, err := client.NewAccountsClient().ListKeys(ctx, resourceGroup, storageAccountName, nil) + if err != nil { + return nil, fmt.Errorf("failed to retrieve storage keys: %w", err) + } + + // There should always be two keys set - this can be thought of as primary/secondary + // so one in-use so other can be regenerated without losing access to the service + s := make([]string, 0) + for i, key := range keys.Keys { + if k := key.Value; k != nil { + s = append(s, *k) + } else { + return nil, fmt.Errorf("storage access key %d not set", i) + } + } + + return &Keys{ + Key1: s[0], + Key2: s[1], + }, nil +} + +func (c *Client) TestHostedZoneLiveness(ctx context.Context, domainName, resourceGroup string) (bool, error) { + client, err := c.newDNSClientFactory() + if err != nil { + return false, err + } + + _, err = client.NewZonesClient().Get(ctx, resourceGroup, domainName, nil) + if err != nil { + //lint:ignore nilerr We cannot tell the difference between a network failure or a missing DNS zone + return false, nil + } + + return true, nil +} + +func NewClient(clientID, clientSecret, subscriptionID, tenantID string) (*Client, error) { + // I don't particularly like this, but there doesn't seem to be any other way + // of achieving this with the SDK. If someone knows a way, please open a PR + // + // @link https://learn.microsoft.com/en-us/azure/developer/go/azure-sdk-authentication-service-principal?tabs=azure-cli + os.Setenv("AZURE_CLIENT_ID", clientID) + os.Setenv("AZURE_CLIENT_SECRET", clientSecret) + os.Setenv("AZURE_TENANT_ID", tenantID) + + cred, err := azidentity.NewDefaultAzureCredential(nil) + if err != nil { + return nil, fmt.Errorf("failed to create default azure credential: %w", err) + } + + return &Client{ + cred: cred, + subscriptionID: subscriptionID, + }, nil +} diff --git a/internal/common.go b/internal/common.go index 910610b9..2c6c9d79 100644 --- a/internal/common.go +++ b/internal/common.go @@ -18,6 +18,8 @@ var SupportedPlatforms = []string{ "akamai-github", "aws-github", "aws-gitlab", + "azure-github", + "azure-gitlab", "civo-github", "civo-gitlab", "digitalocean-github", diff --git a/internal/controller/argocd.go b/internal/controller/argocd.go index 3666864b..96110116 100644 --- a/internal/controller/argocd.go +++ b/internal/controller/argocd.go @@ -39,7 +39,7 @@ func (clctrl *ClusterController) InstallArgoCD() error { if err != nil { return fmt.Errorf("failed to create eks config: %w", err) } - case "akamai", "civo", "digitalocean", "k3s", "vultr": + case "akamai", "azure", "civo", "digitalocean", "k3s", "vultr": kcfg, err = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig) if err != nil { return fmt.Errorf("failed to create Kubernetes config: %w", err) @@ -97,7 +97,7 @@ func (clctrl *ClusterController) InitializeArgoCD() error { if err != nil { return fmt.Errorf("failed to create eks config: %w", err) } - case "akamai", "civo", "digitalocean", "k3s", "vultr": + case "akamai", "azure", "civo", "digitalocean", "k3s", "vultr": kcfg, err = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig) if err != nil { return fmt.Errorf("failed to create Kubernetes config: %w", err) @@ -125,7 +125,7 @@ func (clctrl *ClusterController) InitializeArgoCD() error { var argoCDToken string switch clctrl.CloudProvider { - case "aws", "civo", "google", "digitalocean", "vultr", "k3s": + case "aws", "azure", "civo", "google", "digitalocean", "vultr", "k3s": // kcfg.Clientset.RbacV1(). argoCDStopChannel := make(chan struct{}, 1) @@ -199,7 +199,7 @@ func (clctrl *ClusterController) DeployRegistryApplication() error { if err != nil { return fmt.Errorf("failed to create eks config: %w", err) } - case "akamai", "civo", "digitalocean", "k3s", "vultr": + case "akamai", "azure", "civo", "digitalocean", "k3s", "vultr": kcfg, err = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig) if err != nil { return fmt.Errorf("failed to create Kubernetes config: %w", err) diff --git a/internal/controller/cluster.go b/internal/controller/cluster.go index 05bb5871..34e012c2 100644 --- a/internal/controller/cluster.go +++ b/internal/controller/cluster.go @@ -14,6 +14,7 @@ import ( akamaiext "github.com/konstructio/kubefirst-api/extensions/akamai" awsext "github.com/konstructio/kubefirst-api/extensions/aws" + azureext "github.com/konstructio/kubefirst-api/extensions/azure" civoext "github.com/konstructio/kubefirst-api/extensions/civo" digitaloceanext "github.com/konstructio/kubefirst-api/extensions/digitalocean" googleext "github.com/konstructio/kubefirst-api/extensions/google" @@ -65,6 +66,8 @@ func (clctrl *ClusterController) CreateCluster() error { if err != nil { return fmt.Errorf("failed to update cluster after getting AWS account ID: %w", err) } + case "azure": + tfEnvs = azureext.GetAzureTerraformEnvs(tfEnvs, cl) case "civo": tfEnvs = civoext.GetCivoTerraformEnvs(tfEnvs, cl) case "digitalocean": @@ -141,6 +144,8 @@ func (clctrl *ClusterController) CreateTokens(kind string) interface{} { // provider auth secret gets mapped to these values case "aws": externalDNSProviderTokenEnvName = "not-used-uses-service-account" + case "azure": + externalDNSProviderTokenEnvName = "not-used-uses-managed-service-principal" case "google": // Normally this would be GOOGLE_APPLICATION_CREDENTIALS but we are using a service account instead and // if you set externalDNSProviderTokenEnvName to GOOGLE_APPLICATION_CREDENTIALS then externaldns will overlook the service account @@ -259,6 +264,10 @@ func (clctrl *ClusterController) CreateTokens(kind string) interface{} { // gitopsTemplateTokens.ContainerRegistryURL = fmt.Sprintf("%s/%s", clctrl.ContainerRegistryHost, clctrl.GitAuth.Owner) log.Info().Msgf("NOT using ECR but instead %s URL %s", clctrl.GitProvider, gitopsTemplateTokens.ContainerRegistryURL) } + case "azure": + gitopsTemplateTokens.AzureStorageResourceGroup = fmt.Sprintf("%s-state", clctrl.ClusterName) + gitopsTemplateTokens.AzureStorageContainerName = azureTerraformContainer + gitopsTemplateTokens.AzureDNSZoneResourceGroup = clctrl.AzureDNSZoneResourceGroup case "k3s": gitopsTemplateTokens.K3sServersPrivateIps = clctrl.K3sAuth.K3sServersPrivateIps gitopsTemplateTokens.K3sServersPublicIps = clctrl.K3sAuth.K3sServersPublicIps @@ -298,7 +307,7 @@ func (clctrl *ClusterController) ClusterSecretsBootstrap() error { if err != nil { return fmt.Errorf("failed to create eks config: %w", err) } - case "akamai", "civo", "digitalocean", "k3s", "vultr": + case "akamai", "azure", "civo", "digitalocean", "k3s", "vultr": kcfg, err = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig) if err != nil { return fmt.Errorf("failed to create Kubernetes config during secrets bootstrap: %w", err) @@ -343,6 +352,12 @@ func (clctrl *ClusterController) ClusterSecretsBootstrap() error { log.Error().Msgf("error adding Kubernetes secrets for bootstrap: %s", err) return fmt.Errorf("error adding Kubernetes secrets for bootstrap on aws: %w", err) } + case "azure": + err := azureext.BootstrapAzureMgmtCluster(clientSet, cl, destinationGitopsRepoGitURL) + if err != nil { + log.Error().Msgf("error adding Kubernetes secrets for bootstrap: %s", err) + return fmt.Errorf("error adding Kubernetes secrets for bootstrap on azure: %w", err) + } case "civo": err := civoext.BootstrapCivoMgmtCluster(clientSet, cl, destinationGitopsRepoGitURL) if err != nil { diff --git a/internal/controller/constants.go b/internal/controller/constants.go new file mode 100644 index 00000000..571cf451 --- /dev/null +++ b/internal/controller/constants.go @@ -0,0 +1,3 @@ +package controller + +const azureTerraformContainer = "terraform" diff --git a/internal/controller/controller.go b/internal/controller/controller.go index 772445e7..c4726299 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -12,10 +12,12 @@ import ( "fmt" "net/http" "os" + "regexp" "time" runtime "github.com/konstructio/kubefirst-api/internal" awsinternal "github.com/konstructio/kubefirst-api/internal/aws" + azureinternal "github.com/konstructio/kubefirst-api/internal/azure" "github.com/konstructio/kubefirst-api/internal/constants" "github.com/konstructio/kubefirst-api/internal/env" "github.com/konstructio/kubefirst-api/internal/github" @@ -49,6 +51,7 @@ type ClusterController struct { // auth AkamaiAuth types.AkamaiAuth AWSAuth types.AWSAuth + AzureAuth types.AzureAuth CivoAuth types.CivoAuth DigitaloceanAuth types.DigitaloceanAuth VultrAuth types.VultrAuth @@ -107,8 +110,12 @@ type ClusterController struct { // Telemetry TelemetryEvent telemetry.TelemetryEvent + // Azure + AzureDNSZoneResourceGroup string + // Provider clients AwsClient *awsinternal.Configuration + AzureClient *azureinternal.Client GoogleClient google.Configuration Kcfg *k8s.KubernetesClient Cluster types.Cluster @@ -207,6 +214,7 @@ func (clctrl *ClusterController) InitController(def *types.ClusterDefinition) er clctrl.AkamaiAuth = def.AkamaiAuth clctrl.AWSAuth = def.AWSAuth + clctrl.AzureAuth = def.AzureAuth clctrl.CivoAuth = def.CivoAuth clctrl.DigitaloceanAuth = def.DigitaloceanAuth clctrl.VultrAuth = def.VultrAuth @@ -234,9 +242,22 @@ func (clctrl *ClusterController) InitController(def *types.ClusterDefinition) er } else { clctrl.GitopsTemplateURL = "https://github.com/kubefirst/gitops-template.git" } - if def.CloudProvider == "akamai" { + switch def.CloudProvider { + case "akamai": clctrl.KubefirstStateStoreBucketName = clctrl.ClusterName - } else { + case "azure": + // Azure storage accounts are 3-24 characters and only letters/numbers + maxLen := 24 + reg := regexp.MustCompile(`\W`) + + storeName := fmt.Sprintf("k1%s%s", clusterID, clctrl.ClusterName) + + clctrl.KubefirstStateStoreBucketName = reg.ReplaceAllString(storeName, "") + + if len(clctrl.KubefirstStateStoreBucketName) > maxLen { + clctrl.KubefirstStateStoreBucketName = clctrl.KubefirstStateStoreBucketName[:maxLen] + } + default: clctrl.KubefirstStateStoreBucketName = fmt.Sprintf("k1-state-store-%s-%s", clctrl.ClusterName, clusterID) } @@ -283,6 +304,14 @@ func (clctrl *ClusterController) InitController(def *types.ClusterDefinition) er } clctrl.ProviderConfig = *conf + case "azure": + conf, err := providerConfigs.GetConfig(clctrl.ClusterName, clctrl.DomainName, clctrl.GitProvider, clctrl.GitAuth.Owner, clctrl.GitProtocol, clctrl.CloudflareAuth.APIToken, clctrl.CloudflareAuth.OriginCaIssuerKey) + if err != nil { + return fmt.Errorf("unable to get provider configuration for Azure: %w", err) + } + + clctrl.ProviderConfig = *conf + clctrl.AzureDNSZoneResourceGroup = def.AzureDNSZoneResourceGroup case "civo": conf, err := providerConfigs.GetConfig(clctrl.ClusterName, clctrl.DomainName, clctrl.GitProvider, clctrl.GitAuth.Owner, clctrl.GitProtocol, clctrl.CloudflareAuth.APIToken, clctrl.CloudflareAuth.OriginCaIssuerKey) if err != nil { @@ -348,6 +377,17 @@ func (clctrl *ClusterController) InitController(def *types.ClusterDefinition) er } clctrl.AwsClient = &awsinternal.Configuration{Config: conf} + case "azure": + azureClient, err := azureinternal.NewClient( + clctrl.AzureAuth.ClientID, + clctrl.AzureAuth.ClientSecret, + clctrl.AzureAuth.SubscriptionID, + clctrl.AzureAuth.TenantID, + ) + if err != nil { + return fmt.Errorf("error creating azure client: %w", err) + } + clctrl.AzureClient = azureClient case "google": clctrl.GoogleClient = google.Configuration{ Context: context.Background(), @@ -383,6 +423,7 @@ func (clctrl *ClusterController) InitController(def *types.ClusterDefinition) er KubefirstTeam: clctrl.KubefirstTeam, AkamaiAuth: clctrl.AkamaiAuth, AWSAuth: clctrl.AWSAuth, + AzureAuth: clctrl.AzureAuth, CivoAuth: clctrl.CivoAuth, GoogleAuth: clctrl.GoogleAuth, DigitaloceanAuth: clctrl.DigitaloceanAuth, diff --git a/internal/controller/domain.go b/internal/controller/domain.go index 4c326fdf..adc221bd 100644 --- a/internal/controller/domain.go +++ b/internal/controller/domain.go @@ -40,6 +40,16 @@ func (clctrl *ClusterController) DomainLivenessTest() error { if err != nil { return fmt.Errorf("domain liveness check failed for AWS: %w", err) } + case "azure": + domainLiveness, err := clctrl.AzureClient.TestHostedZoneLiveness(context.Background(), clctrl.DomainName, clctrl.AzureDNSZoneResourceGroup) + if err != nil { + return fmt.Errorf("domain liveness command failed for Azure: %w", err) + } + + err = clctrl.HandleDomainLiveness(domainLiveness) + if err != nil { + return fmt.Errorf("domain liveness check failed for Azure: %w", err) + } case "civo": civoConf := civo.Configuration{ Client: civo.NewCivo(cl.CivoAuth.Token, cl.CloudRegion), diff --git a/internal/controller/git.go b/internal/controller/git.go index 0c303fbd..98eb9f4e 100644 --- a/internal/controller/git.go +++ b/internal/controller/git.go @@ -12,6 +12,7 @@ import ( akamaiext "github.com/konstructio/kubefirst-api/extensions/akamai" awsext "github.com/konstructio/kubefirst-api/extensions/aws" + azureext "github.com/konstructio/kubefirst-api/extensions/azure" civoext "github.com/konstructio/kubefirst-api/extensions/civo" digitaloceanext "github.com/konstructio/kubefirst-api/extensions/digitalocean" googleext "github.com/konstructio/kubefirst-api/extensions/google" @@ -83,6 +84,8 @@ func (clctrl *ClusterController) RunGitTerraform() error { tfEnvs = akamaiext.GetGithubTerraformEnvs(tfEnvs, cl) case "aws": tfEnvs = awsext.GetGithubTerraformEnvs(tfEnvs, cl) + case "azure": + tfEnvs = azureext.GetGithubTerraformEnvs(tfEnvs, cl) case "civo": tfEnvs = civoext.GetGithubTerraformEnvs(tfEnvs, cl) case "google": @@ -100,6 +103,8 @@ func (clctrl *ClusterController) RunGitTerraform() error { tfEnvs = akamaiext.GetGitlabTerraformEnvs(tfEnvs, clctrl.GitlabOwnerGroupID, cl) case "aws": tfEnvs = awsext.GetGitlabTerraformEnvs(tfEnvs, clctrl.GitlabOwnerGroupID, cl) + case "azure": + tfEnvs = azureext.GetGitlabTerraformEnvs(tfEnvs, clctrl.GitlabOwnerGroupID, cl) case "civo": tfEnvs = civoext.GetGitlabTerraformEnvs(tfEnvs, clctrl.GitlabOwnerGroupID, cl) case "google": diff --git a/internal/controller/kubefirst.go b/internal/controller/kubefirst.go index 88e07bac..59fdcdb8 100644 --- a/internal/controller/kubefirst.go +++ b/internal/controller/kubefirst.go @@ -72,7 +72,7 @@ func (clctrl *ClusterController) ExportClusterRecord() error { if err != nil { return fmt.Errorf("failed to create eks config: %w", err) } - case "akamai", "civo", "digitalocean", "k3s", "vultr": + case "akamai", "azure", "civo", "digitalocean", "k3s", "vultr": kcfg, err = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig) if err != nil { return fmt.Errorf("failed to create Kubernetes config: %w", err) diff --git a/internal/controller/repository.go b/internal/controller/repository.go index 485778af..9731252c 100644 --- a/internal/controller/repository.go +++ b/internal/controller/repository.go @@ -86,6 +86,28 @@ func (clctrl *ClusterController) RepositoryPrep() error { if err != nil { return fmt.Errorf("error preparing git repositories for AWS: %w", err) } + case "azure": + err := providerConfigs.PrepareGitRepositories( + clctrl.CloudProvider, + clctrl.GitProvider, + clctrl.ClusterName, + clctrl.ClusterType, + clctrl.ProviderConfig.DestinationGitopsRepoURL, + clctrl.ProviderConfig.GitopsDir, + clctrl.GitopsTemplateBranch, + clctrl.GitopsTemplateURL, + clctrl.ProviderConfig.DestinationMetaphorRepoURL, + clctrl.ProviderConfig.K1Dir, + clctrl.CreateTokens("gitops").(*providerConfigs.GitopsDirectoryValues), // tokens created on the fly + clctrl.ProviderConfig.MetaphorDir, + clctrl.CreateTokens("metaphor").(*providerConfigs.MetaphorTokenValues), // tokens created on the fly + true, + cl.GitProtocol, + useCloudflareOriginIssuer, + ) + if err != nil { + return fmt.Errorf("error preparing git repositories for Azure: %w", err) + } case "civo": err := providerConfigs.PrepareGitRepositories( clctrl.CloudProvider, diff --git a/internal/controller/state.go b/internal/controller/state.go index 6375c436..d93ffac3 100644 --- a/internal/controller/state.go +++ b/internal/controller/state.go @@ -68,6 +68,45 @@ func (clctrl *ClusterController) StateStoreCredentials() error { telemetry.SendEvent(clctrl.TelemetryEvent, telemetry.StateStoreCredentialsCreateFailed, err.Error()) return fmt.Errorf("failed to update cluster after creating AWS state store: %w", err) } + case "azure": + // Azure storage is non-S3 compliant + location := clctrl.CloudRegion + resourceGroup := fmt.Sprintf("%s-state", clctrl.ClusterName) + containerName := azureTerraformContainer + + ctx := context.Background() + + if _, err := clctrl.AzureClient.CreateResourceGroup(ctx, resourceGroup, location); err != nil { + telemetry.SendEvent(clctrl.TelemetryEvent, telemetry.StateStoreCreateFailed, err.Error()) + return fmt.Errorf("error creating azure storage resource group %s: %w", resourceGroup, err) + } + + if _, err := clctrl.AzureClient.CreateStorageAccount( + ctx, + location, + resourceGroup, + clctrl.KubefirstStateStoreBucketName, + ); err != nil { + telemetry.SendEvent(clctrl.TelemetryEvent, telemetry.StateStoreCreateFailed, err.Error()) + return fmt.Errorf("error creating azure storage account %s: %w", clctrl.KubefirstStateStoreBucketName, err) + } + + keys, err := clctrl.AzureClient.GetStorageAccessKeys(ctx, resourceGroup, clctrl.KubefirstStateStoreBucketName) + if err != nil { + telemetry.SendEvent(clctrl.TelemetryEvent, telemetry.StateStoreCreateFailed, err.Error()) + return fmt.Errorf("error retrieving azure storage account keys %s: %w", clctrl.KubefirstStateStoreBucketName, err) + } + + // Azure storage is not S3 compatible, but reusing this struct in a (roughly) synonymous way + stateStoreData = pkgtypes.StateStoreCredentials{ + Name: clctrl.KubefirstStateStoreBucketName, + SecretAccessKey: keys.Key1, + } + + if _, err := clctrl.AzureClient.CreateBlobContainer(ctx, clctrl.KubefirstStateStoreBucketName, containerName); err != nil { + telemetry.SendEvent(clctrl.TelemetryEvent, telemetry.StateStoreCreateFailed, err.Error()) + return fmt.Errorf("error creating blob storage container %s: %w", clctrl.KubefirstStateStoreBucketName, err) + } case "civo": civoConf := civo.Configuration{ Client: civo.NewCivo(cl.CivoAuth.Token, cl.CloudRegion), diff --git a/internal/controller/tools.go b/internal/controller/tools.go index cafd535b..a6553873 100644 --- a/internal/controller/tools.go +++ b/internal/controller/tools.go @@ -52,6 +52,19 @@ func (clctrl *ClusterController) DownloadTools(toolsDir string) error { log.Error().Msgf("error downloading dependencies: %s", err) return fmt.Errorf("failed to download tools for aws: %w", err) } + case "azure": + err := utils.DownloadTools( + clctrl.ProviderConfig.KubectlClient, + providerConfigs.KubectlClientVersion, + providerConfigs.LocalhostOS, + providerConfigs.LocalhostArch, + providerConfigs.TerraformClientVersion, + toolsDir, + ) + if err != nil { + log.Error().Msgf("error downloading dependencies: %s", err) + return fmt.Errorf("failed to download tools for azure: %w", err) + } case "civo": err := utils.DownloadTools( clctrl.ProviderConfig.KubectlClient, diff --git a/internal/controller/users.go b/internal/controller/users.go index d979c9e9..a9341419 100644 --- a/internal/controller/users.go +++ b/internal/controller/users.go @@ -12,6 +12,7 @@ import ( akamaiext "github.com/konstructio/kubefirst-api/extensions/akamai" awsext "github.com/konstructio/kubefirst-api/extensions/aws" + azureext "github.com/konstructio/kubefirst-api/extensions/azure" civoext "github.com/konstructio/kubefirst-api/extensions/civo" digitaloceanext "github.com/konstructio/kubefirst-api/extensions/digitalocean" googleext "github.com/konstructio/kubefirst-api/extensions/google" @@ -40,7 +41,7 @@ func (clctrl *ClusterController) RunUsersTerraform() error { if err != nil { return fmt.Errorf("failed to create eks config: %w", err) } - case "akamai", "civo", "digitalocean", "k3s", "vultr": + case "akamai", "azure", "civo", "digitalocean", "k3s", "vultr": kcfg, err = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig) if err != nil { return fmt.Errorf("failed to create Kubernetes config: %w", err) @@ -66,6 +67,9 @@ func (clctrl *ClusterController) RunUsersTerraform() error { case "aws": tfEnvs = awsext.GetAwsTerraformEnvs(tfEnvs, cl) tfEnvs = awsext.GetUsersTerraformEnvs(kcfg.Clientset, cl, tfEnvs) + case "azure": + tfEnvs = azureext.GetAzureTerraformEnvs(tfEnvs, cl) + tfEnvs = azureext.GetUsersTerraformEnvs(kcfg.Clientset, cl, tfEnvs) case "civo": tfEnvs = civoext.GetCivoTerraformEnvs(tfEnvs, cl) tfEnvs = civoext.GetUsersTerraformEnvs(kcfg.Clientset, cl, tfEnvs) diff --git a/internal/controller/vault.go b/internal/controller/vault.go index afddc3be..f800a9dd 100644 --- a/internal/controller/vault.go +++ b/internal/controller/vault.go @@ -19,6 +19,7 @@ import ( vaultapi "github.com/hashicorp/vault/api" akamaiext "github.com/konstructio/kubefirst-api/extensions/akamai" awsext "github.com/konstructio/kubefirst-api/extensions/aws" + azureext "github.com/konstructio/kubefirst-api/extensions/azure" civoext "github.com/konstructio/kubefirst-api/extensions/civo" digitaloceanext "github.com/konstructio/kubefirst-api/extensions/digitalocean" googleext "github.com/konstructio/kubefirst-api/extensions/google" @@ -75,7 +76,7 @@ func (clctrl *ClusterController) InitializeVault() error { if err != nil { return fmt.Errorf("failed to create eks config: %w", err) } - case "akamai", "civo", "digitalocean", "k3s", "vultr": + case "akamai", "azure", "civo", "digitalocean", "k3s", "vultr": kcfg, err = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig) if err != nil { return fmt.Errorf("failed to create Kubernetes config for vault initialization: %w", err) @@ -119,7 +120,7 @@ func (clctrl *ClusterController) InitializeVault() error { if err != nil { return fmt.Errorf("failed to create vault secret in Kubernetes: %w", err) } - case "akamai", "civo", "digitalocean", "k3s", "vultr": + case "akamai", "azure", "civo", "digitalocean", "k3s", "vultr": // Initialize and unseal Vault // Build and apply manifests yamlData, err := kcfg.KustomizeBuild(vaultHandlerPath) @@ -175,7 +176,7 @@ func (clctrl *ClusterController) RunVaultTerraform() error { if err != nil { return fmt.Errorf("failed to create eks config: %w", err) } - case "akamai", "civo", "digitalocean", "k3s", "vultr": + case "akamai", "azure", "civo", "digitalocean", "k3s", "vultr": kcfg, err = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig) if err != nil { return fmt.Errorf("failed to create Kubernetes config for vault terraform execution: %w", err) @@ -223,6 +224,9 @@ func (clctrl *ClusterController) RunVaultTerraform() error { case "aws": tfEnvs = awsext.GetVaultTerraformEnvs(kcfg.Clientset, cl, tfEnvs) tfEnvs = awsext.GetAwsTerraformEnvs(tfEnvs, cl) + case "azure": + tfEnvs = azureext.GetVaultTerraformEnvs(kcfg.Clientset, cl, tfEnvs) + tfEnvs = azureext.GetAzureTerraformEnvs(tfEnvs, cl) case "civo": tfEnvs = civoext.GetVaultTerraformEnvs(kcfg.Clientset, cl, tfEnvs) tfEnvs = civoext.GetCivoTerraformEnvs(tfEnvs, cl) @@ -296,9 +300,7 @@ func (clctrl *ClusterController) WriteVaultSecrets() error { externalDNSToken = cl.VultrAuth.Token case "digitalocean": externalDNSToken = cl.DigitaloceanAuth.Token - case "aws": - externalDNSToken = "implement with cluster management" - case "google": + case "aws", "azure", "google": externalDNSToken = "implement with cluster management" case "cloudflare": externalDNSToken = cl.CloudflareAuth.APIToken @@ -311,7 +313,7 @@ func (clctrl *ClusterController) WriteVaultSecrets() error { if err != nil { return fmt.Errorf("failed to create eks config: %w", err) } - case "akamai", "civo", "digitalocean", "k3s", "vultr": + case "akamai", "azure", "civo", "digitalocean", "k3s", "vultr": kcfg, err = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig) if err != nil { return fmt.Errorf("failed to create Kubernetes config: %w", err) @@ -399,7 +401,7 @@ func (clctrl *ClusterController) WaitForVault() error { if err != nil { return fmt.Errorf("failed to create eks config: %w", err) } - case "akamai", "civo", "digitalocean", "k3s", "vultr": + case "akamai", "azure", "civo", "digitalocean", "k3s", "vultr": var err error kcfg, err = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig) if err != nil { diff --git a/internal/router/api/v1/cluster.go b/internal/router/api/v1/cluster.go index 39fd3df9..78f0289f 100644 --- a/internal/router/api/v1/cluster.go +++ b/internal/router/api/v1/cluster.go @@ -29,6 +29,7 @@ import ( pkgtypes "github.com/konstructio/kubefirst-api/pkg/types" "github.com/konstructio/kubefirst-api/providers/akamai" "github.com/konstructio/kubefirst-api/providers/aws" + "github.com/konstructio/kubefirst-api/providers/azure" "github.com/konstructio/kubefirst-api/providers/civo" "github.com/konstructio/kubefirst-api/providers/digitalocean" "github.com/konstructio/kubefirst-api/providers/google" @@ -319,6 +320,7 @@ func PostCreateCluster(c *gin.Context) { // Assign cloud and git credentials clusterDefinition.AkamaiAuth = cluster.AkamaiAuth clusterDefinition.AWSAuth = cluster.AWSAuth + clusterDefinition.AzureAuth = cluster.AzureAuth clusterDefinition.CivoAuth = cluster.CivoAuth clusterDefinition.VultrAuth = cluster.VultrAuth clusterDefinition.DigitaloceanAuth = cluster.DigitaloceanAuth @@ -420,6 +422,37 @@ func PostCreateCluster(c *gin.Context) { } }() + c.JSON(http.StatusAccepted, types.JSONSuccessResponse{ + Message: "cluster create enqueued", + }) + case "azure": + if useSecretForAuth { + err := utils.ValidateAuthenticationFields(k1AuthSecret) + if err != nil { + c.JSON(http.StatusBadRequest, types.JSONFailureResponse{ + Message: fmt.Sprintf("error checking azure auth: %s", err), + }) + return + } + clusterDefinition.CivoAuth = pkgtypes.CivoAuth{ + Token: k1AuthSecret["azure-token"], + } + } else if clusterDefinition.AzureAuth.ClientID == "" || + clusterDefinition.AzureAuth.ClientSecret == "" || + clusterDefinition.AzureAuth.SubscriptionID == "" || + clusterDefinition.AzureAuth.TenantID == "" { + c.JSON(http.StatusBadRequest, types.JSONFailureResponse{ + Message: "missing authentication credentials in request, please check and try again", + }) + return + } + go func() { + err = azure.CreateAzureCluster(&clusterDefinition) + if err != nil { + log.Error().Msg(err.Error()) + } + }() + c.JSON(http.StatusAccepted, types.JSONSuccessResponse{ Message: "cluster create enqueued", }) diff --git a/pkg/constants/cloudProviderDefaults.go b/pkg/constants/cloudProviderDefaults.go index 8831f787..8cbc3886 100644 --- a/pkg/constants/cloudProviderDefaults.go +++ b/pkg/constants/cloudProviderDefaults.go @@ -5,6 +5,7 @@ import "github.com/konstructio/kubefirst-api/pkg/types" var cloudProviderDefaults = types.CloudProviderDefaults{ Akamai: types.CloudDefault{InstanceSize: "g6-standard-4", NodeCount: "4"}, Aws: types.CloudDefault{InstanceSize: "m5.large", NodeCount: "5"}, + Azure: types.CloudDefault{InstanceSize: "Standard_D2s_v4", NodeCount: "3"}, Civo: types.CloudDefault{InstanceSize: "g4s.kube.large", NodeCount: "4"}, DigitalOcean: types.CloudDefault{InstanceSize: "s-4vcpu-8gb", NodeCount: "4"}, Google: types.CloudDefault{InstanceSize: "e2-medium", NodeCount: "2"}, diff --git a/pkg/providerConfigs/adjustDriver.go b/pkg/providerConfigs/adjustDriver.go index 18cf7682..a89c099a 100644 --- a/pkg/providerConfigs/adjustDriver.go +++ b/pkg/providerConfigs/adjustDriver.go @@ -27,6 +27,8 @@ const ( AkamaiGitHub = "akamai-github" AwsGitHub = "aws-github" AwsGitLab = "aws-gitlab" + AzureGitHub = "azure-github" + AzureGitLab = "azure-gitlab" CivoGitHub = "civo-github" CivoGitLab = "civo-gitlab" GoogleGitHub = "google-github" @@ -168,6 +170,8 @@ func AdjustGitopsRepo( case AkamaiGitHub, AwsGitHub, AwsGitLab, + AzureGitHub, + AzureGitLab, CivoGitHub, CivoGitLab, GoogleGitHub, diff --git a/pkg/providerConfigs/bootstrapSecrets.go b/pkg/providerConfigs/bootstrapSecrets.go index 7d44fd69..1006eb87 100644 --- a/pkg/providerConfigs/bootstrapSecrets.go +++ b/pkg/providerConfigs/bootstrapSecrets.go @@ -138,9 +138,7 @@ func BootstrapSecrets(client kubernetes.Interface, cl *pkgtypes.Cluster, extraSe externalDNSToken = cl.VultrAuth.Token case "digitalocean": externalDNSToken = cl.DigitaloceanAuth.Token - case "aws": - externalDNSToken = "implement with cluster management" - case "google": + case "aws", "azure", "google": externalDNSToken = "implement with cluster management" case "cloudflare": externalDNSToken = cl.CloudflareAuth.APIToken diff --git a/pkg/providerConfigs/detokenize.go b/pkg/providerConfigs/detokenize.go index 09a97dc6..24585f19 100644 --- a/pkg/providerConfigs/detokenize.go +++ b/pkg/providerConfigs/detokenize.go @@ -87,6 +87,16 @@ func detokenizeGitops(tokens *GitopsDirectoryValues, gitProtocol string, useClou newContents = strings.ReplaceAll(newContents, "", tokens.AwsIamArnAccountRoot) newContents = strings.ReplaceAll(newContents, "", tokens.AwsNodeCapacityType) + // Azure + azureDNSZoneName := "" + if tokens.ExternalDNSProviderName == "azure" { + azureDNSZoneName = fullDomainName + } + newContents = strings.ReplaceAll(newContents, "", tokens.AzureStorageResourceGroup) + newContents = strings.ReplaceAll(newContents, "", tokens.AzureStorageContainerName) + newContents = strings.ReplaceAll(newContents, "", tokens.AzureDNSZoneResourceGroup) + newContents = strings.ReplaceAll(newContents, "", azureDNSZoneName) // This is only set if using Azure for DNS + // google newContents = strings.ReplaceAll(newContents, "", tokens.GoogleProject) newContents = strings.ReplaceAll(newContents, "", tokens.ForceDestroy) diff --git a/pkg/providerConfigs/types.go b/pkg/providerConfigs/types.go index 79a7d83c..a2c56eb5 100644 --- a/pkg/providerConfigs/types.go +++ b/pkg/providerConfigs/types.go @@ -51,6 +51,10 @@ type GitopsDirectoryValues struct { AwsNodeCapacityType string AwsAccountID string + AzureStorageResourceGroup string + AzureStorageContainerName string + AzureDNSZoneResourceGroup string + GoogleAuth string GoogleProject string GoogleUniqueness string diff --git a/pkg/types/auth.go b/pkg/types/auth.go index 16f3dcb8..221724ff 100644 --- a/pkg/types/auth.go +++ b/pkg/types/auth.go @@ -18,6 +18,14 @@ type AWSAuth struct { SessionToken string `bson:"session_token" json:"session_token"` } +// AzureAuth holds necessary auth credentials for interacting with azure +type AzureAuth struct { + ClientID string `bson:"client_id" json:"client_id"` + ClientSecret string `bson:"client_secret" json:"client_secret"` + TenantID string `bson:"tenant_id" json:"tenant_id"` + SubscriptionID string `bson:"subscription_id" json:"subscription_id"` +} + // CivoAuth holds necessary auth credentials for interacting with civo type CivoAuth struct { Token string `bson:"token" json:"token"` diff --git a/pkg/types/cloudDefaults.go b/pkg/types/cloudDefaults.go index a1c279b5..c7053397 100644 --- a/pkg/types/cloudDefaults.go +++ b/pkg/types/cloudDefaults.go @@ -8,6 +8,7 @@ type CloudDefault struct { type CloudProviderDefaults struct { Akamai CloudDefault `json:"akamai"` Aws CloudDefault `json:"aws"` + Azure CloudDefault `json:"azure"` Civo CloudDefault `json:"civo"` DigitalOcean CloudDefault `json:"do"` Google CloudDefault `json:"google"` diff --git a/pkg/types/cluster.go b/pkg/types/cluster.go index 2c31a680..57daf3cc 100644 --- a/pkg/types/cluster.go +++ b/pkg/types/cluster.go @@ -14,7 +14,7 @@ import ( type ClusterDefinition struct { // Cluster AdminEmail string `json:"admin_email" binding:"required"` - CloudProvider string `json:"cloud_provider" binding:"required,oneof=akamai aws civo digitalocean google k3s vultr"` + CloudProvider string `json:"cloud_provider" binding:"required,oneof=akamai aws azure civo digitalocean google k3s vultr"` CloudRegion string `json:"cloud_region" binding:"required"` ClusterName string `json:"cluster_name,omitempty"` DomainName string `json:"domain_name" binding:"required"` @@ -38,9 +38,13 @@ type ClusterDefinition struct { // AWS ECR bool `json:"ecr,omitempty"` + // Azure + AzureDNSZoneResourceGroup string `json:"azure_dns_zone_resource_group,omitempty"` + // Auth AkamaiAuth AkamaiAuth `json:"akamai_auth,omitempty"` AWSAuth AWSAuth `json:"aws_auth,omitempty"` + AzureAuth AzureAuth `json:"azure_auth,omitempty"` CivoAuth CivoAuth `json:"civo_auth,omitempty"` DigitaloceanAuth DigitaloceanAuth `json:"do_auth,omitempty"` VultrAuth VultrAuth `json:"vultr_auth,omitempty"` @@ -76,6 +80,7 @@ type Cluster struct { // Auth AkamaiAuth AkamaiAuth `bson:"akamai_auth,omitempty" json:"akamai_auth,omitempty"` AWSAuth AWSAuth `bson:"aws_auth,omitempty" json:"aws_auth,omitempty"` + AzureAuth AzureAuth `json:"azure_auth,omitempty"` CivoAuth CivoAuth `bson:"civo_auth,omitempty" json:"civo_auth,omitempty"` DigitaloceanAuth DigitaloceanAuth `bson:"do_auth,omitempty" json:"do_auth,omitempty"` VultrAuth VultrAuth `bson:"vultr_auth,omitempty" json:"vultr_auth,omitempty"` @@ -114,6 +119,9 @@ type Cluster struct { AWSKMSKeyID string `bson:"aws_kms_key_id,omitempty" json:"aws_kms_key_id,omitempty"` AWSKMSKeyDetokenizedCheck bool `bson:"aws_kms_key_detokenized_check" json:"aws_kms_key_detokenized_check"` + // Azure + AzureDNSZoneResourceGroup string `bson:"azure_dns_zone_resource_group,omitempty" json:"azure_dns_zone_resource_group,omitempty"` + // Telemetry UseTelemetry bool `bson:"use_telemetry"` diff --git a/pkg/utils/tokens.go b/pkg/utils/tokens.go index 513c6029..9b6d6b43 100644 --- a/pkg/utils/tokens.go +++ b/pkg/utils/tokens.go @@ -148,6 +148,11 @@ func CreateTokensFromDatabaseRecord(cl *pkgtypes.Cluster, registryPath string, s case "aws": gitopsTemplateTokens.KubefirstArtifactsBucket = cl.StateStoreDetails.Name gitopsTemplateTokens.AtlantisWebhookURL = cl.AtlantisWebhookURL + case "azure": + // @todo(sje): is this not used? + gitopsTemplateTokens.AzureStorageResourceGroup = "kubefirst-state" // @todo(sje): take from resourceGroup var in internal/controller/state.go + gitopsTemplateTokens.AzureStorageContainerName = "terraform" // @todo(sje): take from containerName var in internal/controller/state.go + gitopsTemplateTokens.AzureDNSZoneResourceGroup = cl.AzureDNSZoneResourceGroup } return gitopsTemplateTokens diff --git a/providers/azure/create.go b/providers/azure/create.go new file mode 100644 index 00000000..1f97a9d5 --- /dev/null +++ b/providers/azure/create.go @@ -0,0 +1,315 @@ +package azure + +import ( + "fmt" + "os" + + "github.com/konstructio/kubefirst-api/internal/constants" + "github.com/konstructio/kubefirst-api/internal/controller" + "github.com/konstructio/kubefirst-api/internal/secrets" + "github.com/konstructio/kubefirst-api/internal/services" + "github.com/konstructio/kubefirst-api/internal/ssl" + "github.com/konstructio/kubefirst-api/pkg/k8s" + pkgtypes "github.com/konstructio/kubefirst-api/pkg/types" + "github.com/rs/zerolog/log" +) + +func CreateAzureCluster(definition *pkgtypes.ClusterDefinition) error { + ctrl := controller.ClusterController{} + + log.Debug().Msg("initializing controller") + err := ctrl.InitController(definition) + if err != nil { + return fmt.Errorf("error initializing controller: %w", err) + } + + ctrl.Cluster.InProgress = true + log.Debug().Msg("updating cluster secrets") + err = secrets.UpdateCluster(ctrl.KubernetesClient, ctrl.Cluster) + if err != nil { + return fmt.Errorf("error updating cluster secrets: %w", err) + } + + log.Debug().Msg("downling tools") + err = ctrl.DownloadTools(ctrl.ProviderConfig.ToolsDir) + if err != nil { + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error downloading tools: %w", err) + } + + log.Debug().Msg("checking domain liveness") + err = ctrl.DomainLivenessTest() + if err != nil { + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error running domain liveness test: %w", err) + } + + log.Debug().Msg("creating state store") + err = ctrl.StateStoreCredentials() + if err != nil { + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error creating state store: %w", err) + } + + log.Debug().Msg("initializing git") + err = ctrl.GitInit() + if err != nil { + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error initializing git: %w", err) + } + + log.Debug().Msg("initializing bot") + err = ctrl.InitializeBot() + if err != nil { + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error initializing bot: %w", err) + } + + log.Debug().Msg("repository prep") + err = ctrl.RepositoryPrep() + if err != nil { + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error preparing repository: %w", err) + } + + log.Debug().Msg("running git terraform") + err = ctrl.RunGitTerraform() + if err != nil { + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error running git terraform: %w", err) + } + + log.Debug().Msg("pushing repository") + err = ctrl.RepositoryPush() + if err != nil { + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error pushing repository: %w", err) + } + + log.Debug().Msg("pushing repository") + err = ctrl.CreateCluster() + if err != nil { + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error creating cluster: %w", err) + } + + log.Debug().Msg("creating cluster") + err = ctrl.CreateCluster() + if err != nil { + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error creating cluster: %w", err) + } + + log.Debug().Msg("bootstrapping cluster secrets") + err = ctrl.ClusterSecretsBootstrap() + if err != nil { + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error bootstrapping cluster secrets: %w", err) + } + + // * check for ssl restore + log.Info().Msg("checking for tls secrets to restore") + secretsFilesToRestore, err := os.ReadDir(ctrl.ProviderConfig.SSLBackupDir + "/secrets") + if err != nil { + if os.IsNotExist(err) { + log.Info().Msg("no files found in secrets directory, continuing") + } else { + log.Info().Msgf("unable to check for TLS secrets to restore: %s", err.Error()) + } + } + + if len(secretsFilesToRestore) != 0 { + // todo would like these but requires CRD's and is not currently supported + // add crds ( use execShellReturnErrors? ) + // https://raw.githubusercontent.com/cert-manager/cert-manager/v1.11.0/deploy/crds/crd-clusterissuers.yaml + // https://raw.githubusercontent.com/cert-manager/cert-manager/v1.11.0/deploy/crds/crd-certificates.yaml + // add certificates, and clusterissuers + log.Info().Msgf("found %d tls secrets to restore", len(secretsFilesToRestore)) + ssl.Restore(ctrl.ProviderConfig.SSLBackupDir, ctrl.ProviderConfig.Kubeconfig) + } else { + log.Info().Msg("no files found in secrets directory, continuing") + } + + log.Debug().Msg("installing argocd") + err = ctrl.InstallArgoCD() + if err != nil { + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error installing argocd: %w", err) + } + + log.Debug().Msg("initializing argocd") + err = ctrl.InitializeArgoCD() + if err != nil { + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error initializing argocd: %w", err) + } + + log.Debug().Msg("deploying registry application") + err = ctrl.DeployRegistryApplication() + if err != nil { + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error deploying registry application: %w", err) + } + + log.Debug().Msg("waiting for vault readiness") + err = ctrl.WaitForVault() + if err != nil { + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error waiting for vault: %w", err) + } + + log.Debug().Msg("initializing vault") + err = ctrl.InitializeVault() + if err != nil { + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error initializing vault: %w", err) + } + + // Create kubeconfig client + log.Debug().Msg("creating kubeconfig") + kcfg, err := k8s.CreateKubeConfig(false, ctrl.ProviderConfig.Kubeconfig) + if err != nil { + return fmt.Errorf("error creating kubeconfig: %w", err) + } + + // * configure vault with terraform + // * vault port-forward + vaultStopChannel := make(chan struct{}, 1) + defer func() { + close(vaultStopChannel) + }() + k8s.OpenPortForwardPodWrapper( + kcfg.Clientset, + kcfg.RestConfig, + "vault-0", + "vault", + 8200, + 8200, + vaultStopChannel, + ) + + log.Debug().Msg("running vault terraform") + err = ctrl.RunVaultTerraform() + if err != nil { + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error running vault terraform: %w", err) + } + + log.Debug().Msg("write vault secrets") + err = ctrl.WriteVaultSecrets() + if err != nil { + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error writing vault secrets: %w", err) + } + + log.Debug().Msg("running users terraform") + err = ctrl.RunUsersTerraform() + if err != nil { + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error running users terraform: %w", err) + } + + // Wait for last sync wave app transition to Running + log.Info().Msg("waiting for final sync wave Deployment to transition to Running") + crossplaneDeployment, err := k8s.ReturnDeploymentObject( + kcfg.Clientset, + "app.kubernetes.io/instance", + "crossplane", + "crossplane-system", + 3600, + ) + if err != nil { + log.Error().Msgf("Error finding crossplane Deployment: %s", err) + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error finding crossplane Deployment: %w", err) + } + log.Info().Msg("waiting on dns, tls certificates from letsencrypt and remaining sync waves.\n this may take up to 60 minutes but regularly completes in under 20 minutes") + _, err = k8s.WaitForDeploymentReady(kcfg.Clientset, crossplaneDeployment, 3600) + if err != nil { + log.Error().Msgf("Error waiting for all Apps to sync ready state: %s", err) + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error waiting for all Apps to sync ready state: %w", err) + } + + // * export and import cluster + log.Debug().Msg("exporting cluster record") + err = ctrl.ExportClusterRecord() + if err != nil { + log.Error().Msgf("Error exporting cluster record: %s", err) + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error exporting cluster record: %w", err) + } + + // Create default service entries + log.Debug().Msg("getting cluster secrets") + cl, err := secrets.GetCluster(ctrl.KubernetesClient, ctrl.ClusterName) + if err != nil { + log.Error().Msgf("error getting cluster %s: %s", ctrl.ClusterName, err) + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error getting cluster %s: %w", ctrl.ClusterName, err) + } + + log.Debug().Msg("adding default services") + err = services.AddDefaultServices(cl) + if err != nil { + log.Error().Msgf("error adding default service entries for cluster %s: %s", cl.ClusterName, err) + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error adding default service entries for cluster %s: %w", cl.ClusterName, err) + } + + if ctrl.InstallKubefirstPro { + log.Info().Msg("waiting for kubefirst-pro-api Deployment to transition to Running") + kubefirstProAPI, err := k8s.ReturnDeploymentObject( + kcfg.Clientset, + "app.kubernetes.io/name", + "kubefirst-pro-api", + "kubefirst", + 1200, + ) + if err != nil { + log.Error().Msgf("Error finding kubefirst-pro-api Deployment: %s", err) + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error finding kubefirst-pro-api Deployment: %w", err) + } + + _, err = k8s.WaitForDeploymentReady(kcfg.Clientset, kubefirstProAPI, 300) + if err != nil { + log.Error().Msgf("Error waiting for kubefirst-pro-api to transition to Running: %s", err) + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error waiting for kubefirst-pro-api to transition to Running: %w", err) + } + } + + // Wait for last sync wave app transition to Running + log.Info().Msg("waiting for final sync wave Deployment to transition to Running") + argocdDeployment, err := k8s.ReturnDeploymentObject( + kcfg.Clientset, + "app.kubernetes.io/name", + "argocd-server", + "argocd", + 3600, + ) + if err != nil { + log.Error().Msgf("Error finding argocd Deployment: %s", err) + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error finding argocd Deployment: %w", err) + } + _, err = k8s.WaitForDeploymentReady(kcfg.Clientset, argocdDeployment, 3600) + if err != nil { + log.Error().Msgf("Error waiting for argocd deployment to enter Ready state: %s", err) + ctrl.UpdateClusterOnError(err.Error()) + return fmt.Errorf("error waiting for argocd deployment to enter Ready state: %w", err) + } + + ctrl.Cluster.Status = constants.ClusterStatusProvisioned + ctrl.Cluster.InProgress = false + err = secrets.UpdateCluster(ctrl.KubernetesClient, ctrl.Cluster) + if err != nil { + log.Error().Msgf("error updating cluster status: %s", err) + return fmt.Errorf("error updating cluster status: %w", err) + } + + log.Info().Msgf("Azure infrastructure successfully created: %s", definition.ClusterName) + + return nil +}