From 4d99f9088ee01f1109992b7d97bdbf26ff75ea50 Mon Sep 17 00:00:00 2001 From: Wei Lim Date: Mon, 11 Nov 2024 13:55:36 -0800 Subject: [PATCH 1/3] make bicep cache module-path based --- .../pkg/infra/provisioning/bicep/bicep_provider.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cli/azd/pkg/infra/provisioning/bicep/bicep_provider.go b/cli/azd/pkg/infra/provisioning/bicep/bicep_provider.go index 0b379b054c1..389afef1cb7 100644 --- a/cli/azd/pkg/infra/provisioning/bicep/bicep_provider.go +++ b/cli/azd/pkg/infra/provisioning/bicep/bicep_provider.go @@ -73,7 +73,7 @@ type BicepProvider struct { curPrincipal provisioning.CurrentPrincipalIdProvider ignoreDeploymentState bool // compileBicepResult is cached to avoid recompiling the same bicep file multiple times in the same azd run. - compileBicepMemoryCache *compileBicepResult + compileBicepMemoryCache map[string]compileBicepResult keyvaultService keyvault.KeyVaultService portalUrlBase string } @@ -1547,8 +1547,8 @@ type compileBicepResult struct { func (p *BicepProvider) compileBicep( ctx context.Context, modulePath string, ) (*compileBicepResult, error) { - if p.compileBicepMemoryCache != nil { - return p.compileBicepMemoryCache, nil + if val, ok := p.compileBicepMemoryCache[modulePath]; ok { + return &val, nil } var compiled string @@ -1649,13 +1649,14 @@ func (p *BicepProvider) compileBicep( } } } - p.compileBicepMemoryCache = &compileBicepResult{ + result := compileBicepResult{ RawArmTemplate: rawTemplate, Template: template, Parameters: parameters, } - return p.compileBicepMemoryCache, nil + p.compileBicepMemoryCache[modulePath] = result + return &result, nil } func combineMetadata(base map[string]json.RawMessage, override map[string]json.RawMessage) map[string]json.RawMessage { From 898c002f5a3f0f82033e0671fa574c2c5dba0384 Mon Sep 17 00:00:00 2001 From: Wei Lim Date: Mon, 11 Nov 2024 14:00:31 -0800 Subject: [PATCH 2/3] allow for second infra --- .../vsrpc/environment_service_refresh.go | 2 +- cli/azd/pkg/azapi/resource_service.go | 24 +- cli/azd/pkg/azure/tags.go | 5 + cli/azd/pkg/devcenter/provision_provider.go | 5 +- cli/azd/pkg/infra/deployment_manager.go | 28 +- .../provisioning/bicep/bicep_provider.go | 828 +++++++++++------- .../provisioning/bicep/bicep_provider_test.go | 8 +- .../infra/provisioning_progress_display.go | 87 +- .../provisioning_progress_display_test.go | 6 +- 9 files changed, 617 insertions(+), 376 deletions(-) diff --git a/cli/azd/internal/vsrpc/environment_service_refresh.go b/cli/azd/internal/vsrpc/environment_service_refresh.go index d983920a562..e5280fe7ea2 100644 --- a/cli/azd/internal/vsrpc/environment_service_refresh.go +++ b/cli/azd/internal/vsrpc/environment_service_refresh.go @@ -88,7 +88,7 @@ func (s *environmentService) refreshEnvironmentAsync( _ = observer.OnNext(ctx, newInfoProgressMessage("Loading latest deployment information")) - deployment, err := bicepProvider.LastDeployment(ctx) + deployment, err := bicepProvider.LastRootDeployment(ctx) if err != nil { log.Printf("failed to get latest deployment result: %v", err) } else { diff --git a/cli/azd/pkg/azapi/resource_service.go b/cli/azd/pkg/azapi/resource_service.go index 9a9dde8fd64..9c8cb0e67b4 100644 --- a/cli/azd/pkg/azapi/resource_service.go +++ b/cli/azd/pkg/azapi/resource_service.go @@ -2,8 +2,11 @@ package azapi import ( "context" + "errors" "fmt" + "slices" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" "github.com/azure/azure-dev/cli/azd/pkg/account" @@ -194,6 +197,11 @@ func (rs *ResourceService) DeleteResourceGroup(ctx context.Context, subscription } poller, err := client.BeginDelete(ctx, resourceGroupName, nil) + var respErr *azcore.ResponseError + if errors.As(err, &respErr) && respErr.StatusCode == 404 { // Resource group is already deleted + return nil + } + if err != nil { return fmt.Errorf("beginning resource group deletion: %w", err) } @@ -259,12 +267,16 @@ func GroupByResourceGroup(resources []*armresources.ResourceReference) (map[stri resourceType := resourceId.ResourceType.String() if resourceType != string(AzureResourceTypeResourceGroup) { - groupResources = append(groupResources, &Resource{ - Id: *resource.ID, - Name: resourceId.Name, - Type: resourceType, - Location: resourceId.Location, - }) + if !slices.ContainsFunc(groupResources, func(r *Resource) bool { + return r.Id == *resource.ID + }) { + groupResources = append(groupResources, &Resource{ + Id: *resource.ID, + Name: resourceId.Name, + Type: resourceType, + Location: resourceId.Location, + }) + } } resourceMap[resourceId.ResourceGroupName] = groupResources diff --git a/cli/azd/pkg/azure/tags.go b/cli/azd/pkg/azure/tags.go index 96eceb3993a..9a541973373 100644 --- a/cli/azd/pkg/azure/tags.go +++ b/cli/azd/pkg/azure/tags.go @@ -7,6 +7,11 @@ const ( // TagKeyAzdEnvName is the name of the key in the tags map of a resource // used to store the azd environment a resource is associated with. TagKeyAzdEnvName = "azd-env-name" + + // TagKeyAzdModuleName is the name of the key in the tags map of a resource + // used to store the Bicep module a resource is associated with. + TagKeyAzdModuleName = "azd-module-name" + /* #nosec G101 - Potential hardcoded credentials - false positive */ // TagKeyAzdDeploymentStateParamHashName is the name of the key in the tags map of a deployment // used to store the parameters hash. diff --git a/cli/azd/pkg/devcenter/provision_provider.go b/cli/azd/pkg/devcenter/provision_provider.go index 7ae15bdeaac..03a709fee60 100644 --- a/cli/azd/pkg/devcenter/provision_provider.go +++ b/cli/azd/pkg/devcenter/provision_provider.go @@ -475,7 +475,8 @@ func (p *ProvisionProvider) pollForProgress(ctx context.Context, deployment infr } // Report incremental progress - progressDisplay := p.deploymentManager.ProgressDisplay(deployment) + demoMode, _ := strconv.ParseBool(os.Getenv("AZD_DEMO_MODE")) + progressDisplay := p.deploymentManager.ProgressDisplay(demoMode, 1) initialDelay := 3 * time.Second regularDelay := 10 * time.Second @@ -488,7 +489,7 @@ func (p *ProvisionProvider) pollForProgress(ctx context.Context, deployment infr timer.Stop() return case <-timer.C: - if err := progressDisplay.ReportProgress(ctx, &queryStartTime); err != nil { + if err := progressDisplay.ReportProgress(ctx, deployment, &queryStartTime); err != nil { // We don't want to fail the whole deployment if a progress reporting error occurs log.Printf("error while reporting progress: %s", err.Error()) } diff --git a/cli/azd/pkg/infra/deployment_manager.go b/cli/azd/pkg/infra/deployment_manager.go index a6311669370..6e4c46f4e53 100644 --- a/cli/azd/pkg/infra/deployment_manager.go +++ b/cli/azd/pkg/infra/deployment_manager.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "log" "slices" "strings" @@ -47,8 +48,8 @@ func (dm *DeploymentManager) CalculateTemplateHash( return dm.deploymentService.CalculateTemplateHash(ctx, subscriptionId, template) } -func (dm *DeploymentManager) ProgressDisplay(deployment Deployment) *ProvisioningProgressDisplay { - return NewProvisioningProgressDisplay(dm.resourceManager, dm.console, deployment) +func (dm *DeploymentManager) ProgressDisplay(demoMode bool, totalDeployments int) *ProvisioningProgressDisplay { + return NewProvisioningProgressDisplay(dm.resourceManager, dm.console, demoMode, totalDeployments) } func (dm *DeploymentManager) SubscriptionScope(subscriptionId string, location string) *SubscriptionScope { @@ -77,6 +78,7 @@ func (dm *DeploymentManager) CompletedDeployments( ctx context.Context, scope Scope, envName string, + moduleName string, hint string, ) ([]*azapi.ResourceDeployment, error) { deployments, err := scope.ListDeployments(ctx) @@ -94,7 +96,8 @@ func (dm *DeploymentManager) CompletedDeployments( } // Environment matching strategy - // 1. Deployment with azd tagged env name + // 1. Deployment with azd tagged env name + module name + // 1.1 Deployment with azd tagged env name (legacy) // 2. Exact match on environment name to deployment name (old azd strategy) // 3. Multiple matching names based on specified hint (show user prompt) matchingDeployments := []*azapi.ResourceDeployment{} @@ -106,8 +109,23 @@ func (dm *DeploymentManager) CompletedDeployments( continue } - // Match on current azd strategy (tags) or old azd strategy (deployment name) - if v, has := deployment.Tags[azure.TagKeyAzdEnvName]; has && *v == envName || deployment.Name == envName { + // Match on current azd strategy (tags) + if v, has := deployment.Tags[azure.TagKeyAzdEnvName]; has && *v == envName { + moduleVal, hasModuleTag := deployment.Tags[azure.TagKeyAzdModuleName] + if hasModuleTag && *moduleVal == moduleName { + log.Printf("completedDeployments: matched deployment '%s' using moduleName: %s", deployment.Name, moduleName) + return []*azapi.ResourceDeployment{deployment}, nil + } + + // LEGACY: the deployment is of an old format (no module tag), return the deployment based only on env tag + if !hasModuleTag { + log.Printf("completedDeployments: matched deployment '%s' using envName", deployment.Name) + return []*azapi.ResourceDeployment{deployment}, nil + } + } + + // LEGACY: match on deployment name + if deployment.Name == envName { return []*azapi.ResourceDeployment{deployment}, nil } diff --git a/cli/azd/pkg/infra/provisioning/bicep/bicep_provider.go b/cli/azd/pkg/infra/provisioning/bicep/bicep_provider.go index 389afef1cb7..d0bb53f64c1 100644 --- a/cli/azd/pkg/infra/provisioning/bicep/bicep_provider.go +++ b/cli/azd/pkg/infra/provisioning/bicep/bicep_provider.go @@ -23,6 +23,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/cognitiveservices/armcognitiveservices" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" "github.com/azure/azure-dev/cli/azd/pkg/account" "github.com/azure/azure-dev/cli/azd/pkg/async" "github.com/azure/azure-dev/cli/azd/pkg/azapi" @@ -116,60 +117,62 @@ var ErrEnsureEnvPreReqBicepCompileFailed = errors.New("") // EnsureEnv ensures that the environment is in a provision-ready state with required values set, prompting the user if // values are unset. This also requires that the Bicep module can be compiled. func (p *BicepProvider) EnsureEnv(ctx context.Context) error { - modulePath := p.modulePath() + modulePaths := p.modulePaths() - // for .bicepparam, we first prompt for environment values before calling compiling bicepparam file - // which can reference these values - if isBicepParamFile(modulePath) { - if err := provisioning.EnsureSubscriptionAndLocation(ctx, p.envManager, p.env, p.prompters, nil); err != nil { - return err + for _, modulePath := range modulePaths { + // for .bicepparam, we first prompt for environment values before calling compiling bicepparam file + // which can reference these values + if isBicepParamFile(modulePath) { + if err := provisioning.EnsureSubscriptionAndLocation(ctx, p.envManager, p.env, p.prompters, nil); err != nil { + return err + } } - } - compileResult, compileErr := p.compileBicep(ctx, modulePath) - if compileErr != nil { - return fmt.Errorf("%w%w", ErrEnsureEnvPreReqBicepCompileFailed, compileErr) - } + compileResult, compileErr := p.compileBicep(ctx, modulePath) + if compileErr != nil { + return fmt.Errorf("%w%w", ErrEnsureEnvPreReqBicepCompileFailed, compileErr) + } - // for .bicep, azd must load a parameters.json file and create the ArmParameters - if isBicepFile(modulePath) { - var filterLocation = func(loc account.Location) bool { - if locationParam, defined := compileResult.Template.Parameters["location"]; defined { - if locationParam.AllowedValues != nil { - return slices.IndexFunc(*locationParam.AllowedValues, func(allowedValue any) bool { - allowedValueString, goodCast := allowedValue.(string) - return goodCast && loc.Name == allowedValueString - }) != -1 + // for .bicep, azd must load a parameters.json file and create the ArmParameters + if isBicepFile(modulePath) { + var filterLocation = func(loc account.Location) bool { + if locationParam, defined := compileResult.Template.Parameters["location"]; defined { + if locationParam.AllowedValues != nil { + return slices.IndexFunc(*locationParam.AllowedValues, func(allowedValue any) bool { + allowedValueString, goodCast := allowedValue.(string) + return goodCast && loc.Name == allowedValueString + }) != -1 + } } + return true } - return true - } - err := provisioning.EnsureSubscriptionAndLocation(ctx, p.envManager, p.env, p.prompters, filterLocation) - if err != nil { - return err + err := provisioning.EnsureSubscriptionAndLocation(ctx, p.envManager, p.env, p.prompters, filterLocation) + if err != nil { + return err + } + + if _, err := p.ensureParameters(ctx, compileResult.Template); err != nil { + return err + } } - if _, err := p.ensureParameters(ctx, compileResult.Template); err != nil { + scope, err := compileResult.Template.TargetScope() + if err != nil { return err } - } - - scope, err := compileResult.Template.TargetScope() - if err != nil { - return err - } - if scope == azure.DeploymentScopeResourceGroup { - if p.env.Getenv(environment.ResourceGroupEnvVarName) == "" { - rgName, err := p.prompters.PromptResourceGroup(ctx) - if err != nil { - return err - } + if scope == azure.DeploymentScopeResourceGroup { + if p.env.Getenv(environment.ResourceGroupEnvVarName) == "" { + rgName, err := p.prompters.PromptResourceGroup(ctx) + if err != nil { + return err + } - p.env.DotenvSet(environment.ResourceGroupEnvVarName, rgName) - if err := p.envManager.Save(ctx, p.env); err != nil { - return fmt.Errorf("saving resource group name: %w", err) + p.env.DotenvSet(environment.ResourceGroupEnvVarName, rgName) + if err := p.envManager.Save(ctx, p.env); err != nil { + return fmt.Errorf("saving resource group name: %w", err) + } } } } @@ -177,8 +180,8 @@ func (p *BicepProvider) EnsureEnv(ctx context.Context) error { return nil } -func (p *BicepProvider) LastDeployment(ctx context.Context) (*azapi.ResourceDeployment, error) { - modulePath := p.modulePath() +func (p *BicepProvider) LastRootDeployment(ctx context.Context) (*azapi.ResourceDeployment, error) { + modulePath := p.rootModulePath() compileResult, err := p.compileBicep(ctx, modulePath) if err != nil { return nil, fmt.Errorf("compiling bicep template: %w", err) @@ -189,7 +192,8 @@ func (p *BicepProvider) LastDeployment(ctx context.Context) (*azapi.ResourceDepl return nil, fmt.Errorf("computing deployment scope: %w", err) } - return p.latestDeploymentResult(ctx, scope) + moduleName := moduleNameFromPaths(modulePath, p.rootModulePath()) + return p.latestDeploymentResult(ctx, scope, moduleName) } func (p *BicepProvider) State(ctx context.Context, options *provisioning.StateOptions) (*provisioning.StateResult, error) { @@ -211,103 +215,113 @@ func (p *BicepProvider) State(ctx context.Context, options *provisioning.StateOp var outputs azure.ArmTemplateOutputs var scopeErr error - modulePath := p.modulePath() - if _, err := os.Stat(modulePath); err == nil { - compileResult, err := p.compileBicep(ctx, modulePath) - if err != nil { - return nil, fmt.Errorf("compiling bicep template: %w", err) - } + modulePaths := p.modulePaths() + rootModulePath := p.rootModulePath() - scope, err = p.scopeForTemplate(compileResult.Template) - if err != nil { - return nil, fmt.Errorf("computing deployment scope: %w", err) - } + state := provisioning.State{ + Resources: []provisioning.Resource{}, + Outputs: map[string]provisioning.OutputParameter{}, + } + for _, modulePath := range modulePaths { + if _, err := os.Stat(modulePath); err == nil { + compileResult, err := p.compileBicep(ctx, modulePath) + if err != nil { + return nil, fmt.Errorf("compiling bicep template: %w", err) + } - outputs = compileResult.Template.Outputs - } else if errors.Is(err, os.ErrNotExist) { - // To support BYOI (bring your own infrastructure) - // We need to support the case where there template does not contain an `infra` folder. - scope, scopeErr = p.inferScopeFromEnv() - if scopeErr != nil { - return nil, fmt.Errorf("computing deployment scope: %w", err) - } + scope, err = p.scopeForTemplate(compileResult.Template) + if err != nil { + return nil, fmt.Errorf("computing deployment scope: %w", err) + } - outputs = azure.ArmTemplateOutputs{} - } + outputs = compileResult.Template.Outputs + } else if errors.Is(err, os.ErrNotExist) { + // To support BYOI (bring your own infrastructure) + // We need to support the case where there template does not contain an `infra` folder. + scope, scopeErr = p.inferScopeFromEnv() + if scopeErr != nil { + return nil, fmt.Errorf("computing deployment scope: %w", err) + } - spinnerMessage = "Retrieving Azure deployment" - p.console.ShowSpinner(ctx, spinnerMessage, input.Step) + outputs = azure.ArmTemplateOutputs{} + } - var deployment *azapi.ResourceDeployment + spinnerMessage = "Retrieving Azure deployment" + p.console.ShowSpinner(ctx, spinnerMessage, input.Step) - deployments, err := p.deploymentManager.CompletedDeployments(ctx, scope, p.env.Name(), options.Hint()) - p.console.StopSpinner(ctx, "", input.StepDone) + var deployment *azapi.ResourceDeployment - if err != nil { - p.console.StopSpinner(ctx, spinnerMessage, input.StepFailed) - return nil, fmt.Errorf("retrieving deployment: %w", err) - } else { + moduleName := moduleNameFromPaths(modulePath, rootModulePath) + deployments, err := p.deploymentManager.CompletedDeployments(ctx, scope, p.env.Name(), moduleName, options.Hint()) p.console.StopSpinner(ctx, "", input.StepDone) - } - if len(deployments) > 1 { - deploymentOptions := getDeploymentOptions(deployments) + if err != nil { + p.console.StopSpinner(ctx, spinnerMessage, input.StepFailed) + return nil, fmt.Errorf("retrieving deployment: %w", err) + } else { + p.console.StopSpinner(ctx, "", input.StepDone) + } + + if len(deployments) > 1 { + deploymentOptions := getDeploymentOptions(deployments) + + p.console.Message(ctx, output.WithWarningFormat("WARNING: Multiple matching deployments were found\n")) + + promptConfig := input.ConsoleOptions{ + Message: "Select a deployment to continue:", + Options: deploymentOptions, + } - p.console.Message(ctx, output.WithWarningFormat("WARNING: Multiple matching deployments were found\n")) + selectedDeployment, err := p.console.Select(ctx, promptConfig) + if err != nil { + return nil, err + } - promptConfig := input.ConsoleOptions{ - Message: "Select a deployment to continue:", - Options: deploymentOptions, + deployment = deployments[selectedDeployment] + p.console.Message(ctx, "") + } else { + deployment = deployments[0] } - selectedDeployment, err := p.console.Select(ctx, promptConfig) + azdDeployment, err := p.createDeploymentFromArmDeployment(scope, deployment.Name) if err != nil { return nil, err } - deployment = deployments[selectedDeployment] - p.console.Message(ctx, "") - } else { - deployment = deployments[0] - } + p.console.MessageUxItem(ctx, &ux.DoneMessage{ + Message: fmt.Sprintf("Retrieving Azure deployment (%s)", output.WithHighLightFormat(deployment.Name)), + }) - azdDeployment, err := p.createDeploymentFromArmDeployment(scope, deployment.Name) - if err != nil { - return nil, err - } + for _, res := range deployment.Resources { + state.Resources = append(state.Resources, provisioning.Resource{ + Id: *res.ID, + }) + } - p.console.MessageUxItem(ctx, &ux.DoneMessage{ - Message: fmt.Sprintf("Retrieving Azure deployment (%s)", output.WithHighLightFormat(deployment.Name)), - }) + outputs := p.createOutputParameters( + outputs, + azapi.CreateDeploymentOutput(deployment.Outputs), + ) - state := provisioning.State{} - state.Resources = make([]provisioning.Resource, len(deployment.Resources)) + for key, value := range outputs { + state.Outputs[key] = value + } - for idx, res := range deployment.Resources { - state.Resources[idx] = provisioning.Resource{ - Id: *res.ID, + outputsUrl, err := azdDeployment.OutputsUrl(ctx) + if err != nil { + return nil, err } - } - state.Outputs = p.createOutputParameters( - outputs, - azapi.CreateDeploymentOutput(deployment.Outputs), - ) + p.console.Message(ctx, fmt.Sprintf( + "\nFetched Azure infrastructure deployment: %s", + output.WithHyperlink(outputsUrl, deployment.Name), + )) + } p.console.MessageUxItem(ctx, &ux.DoneMessage{ Message: fmt.Sprintf("Updated %d environment variables", len(state.Outputs)), }) - outputsUrl, err := azdDeployment.OutputsUrl(ctx) - if err != nil { - return nil, err - } - - p.console.Message(ctx, fmt.Sprintf( - "\nPopulated environment from Azure infrastructure deployment: %s", - output.WithHyperlink(outputsUrl, deployment.Name), - )) - return &provisioning.StateResult{ State: &state, }, nil @@ -342,10 +356,9 @@ func isBicepParamFile(modulePath string) bool { } // Plans the infrastructure provisioning -func (p *BicepProvider) plan(ctx context.Context) (*deploymentDetails, error) { +func (p *BicepProvider) plan(ctx context.Context, modulePath string) (*deploymentDetails, error) { p.console.ShowSpinner(ctx, "Creating a deployment plan", input.Step) - modulePath := p.modulePath() compileResult, err := p.compileBicep(ctx, modulePath) if err != nil { return nil, fmt.Errorf("creating template: %w", err) @@ -365,7 +378,12 @@ func (p *BicepProvider) plan(ctx context.Context) (*deploymentDetails, error) { return nil, err } - target, err := p.deploymentFromScopeType(deploymentScope) + name := p.env.Name() + moduleName := moduleNameFromPaths(modulePath, p.rootModulePath()) + if moduleName != "" { + name += "-" + moduleName + } + target, err := p.deploymentFromScopeType(deploymentScope, name) if err != nil { return nil, err } @@ -376,8 +394,10 @@ func (p *BicepProvider) plan(ctx context.Context) (*deploymentDetails, error) { }, nil } -func (p *BicepProvider) deploymentFromScopeType(deploymentScopeType azure.DeploymentScope) (infra.Deployment, error) { - deploymentName := p.deploymentManager.GenerateDeploymentName(p.env.Name()) +func (p *BicepProvider) deploymentFromScopeType( + deploymentScopeType azure.DeploymentScope, + baseName string) (infra.Deployment, error) { + deploymentName := p.deploymentManager.GenerateDeploymentName(baseName) if deploymentScopeType == azure.DeploymentScopeSubscription { scope := p.deploymentManager.SubscriptionScope(p.env.GetSubscriptionId(), p.env.GetLocation()) @@ -395,16 +415,34 @@ func (p *BicepProvider) deploymentFromScopeType(deploymentScopeType azure.Deploy return nil, fmt.Errorf("unsupported scope: %s", deploymentScopeType) } +// moduleNameFromPaths returns the module name from the provided paths. +// This name is suitable for use in Azure deployments. +// It returns empty if the modulePath is the root module. +func moduleNameFromPaths(modulePath string, rootModulePath string) string { + rootModuleDir := filepath.Dir(rootModulePath) + moduleDir := filepath.Dir(modulePath) + if !strings.HasPrefix(moduleDir, rootModuleDir) { + panic(fmt.Sprintf("moduleDir %s does not start with rootModuleDir %s", moduleDir, rootModuleDir)) + } + + subModulePath := moduleDir[len(rootModuleDir):] + subModulePath = strings.TrimPrefix(subModulePath, string(os.PathSeparator)) + + // Replace all path separators with a dash to allow use for Azure deployment names + name := strings.ReplaceAll(subModulePath, string(os.PathSeparator), "-") + return name +} + // deploymentState returns the latests deployment if it is the same as the deployment within deploymentData or an error // otherwise. func (p *BicepProvider) deploymentState( ctx context.Context, deploymentData *deploymentDetails, + moduleName string, currentParamsHash string, ) (*azapi.ResourceDeployment, error) { - p.console.ShowSpinner(ctx, "Comparing deployment state", input.Step) - prevDeploymentResult, err := p.latestDeploymentResult(ctx, deploymentData.Target) + prevDeploymentResult, err := p.latestDeploymentResult(ctx, deploymentData.Target, moduleName) if err != nil { return nil, fmt.Errorf("deployment state error: %w", err) } @@ -436,8 +474,9 @@ func (p *BicepProvider) deploymentState( func (p *BicepProvider) latestDeploymentResult( ctx context.Context, scope infra.Scope, + moduleName string, ) (*azapi.ResourceDeployment, error) { - deployments, err := p.deploymentManager.CompletedDeployments(ctx, scope, p.env.Name(), "") + deployments, err := p.deploymentManager.CompletedDeployments(ctx, scope, p.env.Name(), moduleName, "") // findCompletedDeployments returns error if no deployments are found // No need to check for empty list if err != nil { @@ -521,171 +560,258 @@ func (p *BicepProvider) Deploy(ctx context.Context) (*provisioning.DeployResult, logDS("Azure Deployment State is disabled by --no-state arg.") } - bicepDeploymentData, err := p.plan(ctx) - if err != nil { - return nil, err - } + rootModulePath := p.rootModulePath() + modulePaths := p.modulePaths() - deployment, err := p.convertToDeployment(bicepDeploymentData.CompiledBicep.Template) - if err != nil { - return nil, err - } + // plannedDeployment is a deployment that is planned to be executed + type plannedDeployment struct { + *provisioning.Deployment + moduleName string + details *deploymentDetails + hash string - // parameters hash is required for doing deployment state validation check but also to set the hash - // after a successful deployment. - currentParamsHash, parametersHashErr := parametersHash( - bicepDeploymentData.CompiledBicep.Template.Parameters, bicepDeploymentData.CompiledBicep.Parameters) - if parametersHashErr != nil { - // fail to hash parameters won't stop the operation. It only disables deployment state and recording parameters hash - logDS("%s", parametersHashErr.Error()) + skippedFromState bool } - if !p.ignoreDeploymentState && parametersHashErr == nil { - deploymentState, err := p.deploymentState(ctx, bicepDeploymentData, currentParamsHash) - if err == nil { - deployment.Outputs = p.createOutputParameters( - bicepDeploymentData.CompiledBicep.Template.Outputs, - azapi.CreateDeploymentOutput(deploymentState.Outputs), - ) + deployments := make([]*plannedDeployment, len(modulePaths)) - return &provisioning.DeployResult{ - Deployment: deployment, - SkippedReason: provisioning.DeploymentStateSkipped, - }, nil - } - logDS("%s", err.Error()) - } - - cancelProgress := make(chan bool) - defer func() { cancelProgress <- true }() - go func() { - // Disable reporting progress if needed - if use, err := strconv.ParseBool(os.Getenv("AZD_DEBUG_PROVISION_PROGRESS_DISABLE")); err == nil && use { - log.Println("Disabling progress reporting since AZD_DEBUG_PROVISION_PROGRESS_DISABLE was set") - <-cancelProgress - return - } - - // Report incremental progress - progressDisplay := p.deploymentManager.ProgressDisplay(bicepDeploymentData.Target) - // Make initial delay shorter to be more responsive in displaying initial progress - initialDelay := 3 * time.Second - regularDelay := 10 * time.Second - timer := time.NewTimer(initialDelay) - queryStartTime := time.Now() - - for { - select { - case <-cancelProgress: - timer.Stop() - return - case <-timer.C: - if err := progressDisplay.ReportProgress(ctx, &queryStartTime); err != nil { - // We don't want to fail the whole deployment if a progress reporting error occurs - log.Printf("error while reporting progress: %s", err.Error()) - } + // plan all deployments + for i, modulePath := range modulePaths { + pd := plannedDeployment{} + pd.moduleName = moduleNameFromPaths(modulePath, rootModulePath) + + plannedBicep, err := p.plan(ctx, modulePath) + if err != nil { + return nil, err + } + pd.details = plannedBicep + + deployment, err := p.convertToDeployment(plannedBicep.CompiledBicep.Template) + if err != nil { + return nil, err + } + pd.Deployment = deployment + + // parameters hash is required for doing deployment state validation check but also to set the hash + // after a successful deployment. + currentParamsHash, parametersHashErr := parametersHash( + plannedBicep.CompiledBicep.Template.Parameters, plannedBicep.CompiledBicep.Parameters) + if parametersHashErr != nil { + // Failing to hash parameters doesn't stop the provisioning. + // It only disables deployment state and recording parameters hash + logDS("%s", parametersHashErr.Error()) + } + pd.hash = currentParamsHash + + if !p.ignoreDeploymentState && parametersHashErr == nil { + moduleName := moduleNameFromPaths(modulePath, rootModulePath) + deploymentState, err := p.deploymentState(ctx, plannedBicep, moduleName, currentParamsHash) + if err == nil { + deployment.Outputs = p.createOutputParameters( + plannedBicep.CompiledBicep.Template.Outputs, + azapi.CreateDeploymentOutput(deploymentState.Outputs), + ) + pd.skippedFromState = true + deployments[i] = &pd + continue + } + logDS("%s", err.Error()) + } + + deployments[i] = &pd + } - timer.Reset(regularDelay) + // Disable reporting progress if needed + var progressDisplay *infra.ProvisioningProgressDisplay + if use, err := strconv.ParseBool(os.Getenv("AZD_DEBUG_PROVISION_PROGRESS_DISABLE")); err == nil && use { + log.Println("Disabling progress reporting since AZD_DEBUG_PROVISION_PROGRESS_DISABLE was set") + } else { + demoMode, _ := strconv.ParseBool(os.Getenv("AZD_DEMO_MODE")) + totalDeployments := 0 + for _, p := range deployments { + if !p.skippedFromState { + totalDeployments++ } } - }() - // Start the deployment - p.console.ShowSpinner(ctx, "Creating/Updating resources", input.Step) + if totalDeployments > 0 { + progressDisplay = p.deploymentManager.ProgressDisplay(demoMode, totalDeployments) + } + } + + var activeDeploymentDisplay chan infra.Deployment + if progressDisplay != nil { + activeDeploymentDisplay = make(chan infra.Deployment) + cancelProgress := make(chan bool) + defer func() { cancelProgress <- true }() + go func() { + // Make initial delay shorter to be more responsive in displaying initial progress + initialDelay := 3 * time.Second + regularDelay := 10 * time.Second + timer := time.NewTimer(initialDelay) + queryStartTime := time.Now() + deployment := <-activeDeploymentDisplay + + for { + select { + case deployment = <-activeDeploymentDisplay: // update the deployment to report progress for + case <-cancelProgress: + timer.Stop() + return + case <-timer.C: + if err := progressDisplay.ReportProgress(ctx, deployment, &queryStartTime); err != nil { + // We don't want to fail the whole deployment if a progress reporting error occurs + log.Printf("error while reporting progress: %s", err.Error()) + } - deploymentTags := map[string]*string{ - azure.TagKeyAzdEnvName: to.Ptr(p.env.Name()), + timer.Reset(regularDelay) + } + } + }() } - if parametersHashErr == nil { - deploymentTags[azure.TagKeyAzdDeploymentStateParamHashName] = to.Ptr(currentParamsHash) + + p.console.ShowSpinner(ctx, "Creating/Updating resources", input.Step) + for _, deployment := range deployments { + if deployment.skippedFromState { + log.Printf( + "Deployment '%s' skipped, restored outputs: %+v", + deployment.moduleName, + deployment.Outputs) + continue + } + bicepDeploymentData := deployment.details + + deploymentTags := map[string]*string{ + azure.TagKeyAzdEnvName: to.Ptr(p.env.Name()), + azure.TagKeyAzdModuleName: to.Ptr(deployment.moduleName), + } + + if deployment.hash != "" { + deploymentTags[azure.TagKeyAzdDeploymentStateParamHashName] = to.Ptr(deployment.hash) + } + + optionsMap, err := convert.ToMap(p.options) + if err != nil { + return nil, err + } + + if activeDeploymentDisplay != nil { + activeDeploymentDisplay <- bicepDeploymentData.Target + } + + deployResult, err := bicepDeploymentData.Target.Deploy( + ctx, + bicepDeploymentData.CompiledBicep.RawArmTemplate, + bicepDeploymentData.CompiledBicep.Parameters, + deploymentTags, + optionsMap, + ) + if err != nil { + return nil, err + } + + deployment.Outputs = p.createOutputParameters( + bicepDeploymentData.CompiledBicep.Template.Outputs, + azapi.CreateDeploymentOutput(deployResult.Outputs), + ) + + log.Printf( + "Deployment '%s' completed with outputs: %+v", + deployment.moduleName, + deployment.Outputs) } - optionsMap, err := convert.ToMap(p.options) - if err != nil { - return nil, err + result := &provisioning.DeployResult{} + merged := &provisioning.Deployment{ + Parameters: make(map[string]provisioning.InputParameter), + Outputs: make(map[string]provisioning.OutputParameter), } - deployResult, err := p.deployModule( - ctx, - bicepDeploymentData.Target, - bicepDeploymentData.CompiledBicep.RawArmTemplate, - bicepDeploymentData.CompiledBicep.Parameters, - deploymentTags, - optionsMap, - ) - if err != nil { - return nil, err + allSkipped := true + // Merge all deployments + for _, deployment := range deployments { + for key, value := range deployment.Outputs { + merged.Outputs[key] = value + } + + allSkipped = allSkipped && deployment.skippedFromState } - deployment.Outputs = p.createOutputParameters( - bicepDeploymentData.CompiledBicep.Template.Outputs, - azapi.CreateDeploymentOutput(deployResult.Outputs), - ) + result.Deployment = merged + if allSkipped { + result.SkippedReason = provisioning.DeploymentStateSkipped + } - return &provisioning.DeployResult{ - Deployment: deployment, - }, nil + return result, nil } // Preview runs deploy using the what-if argument func (p *BicepProvider) Preview(ctx context.Context) (*provisioning.DeployPreviewResult, error) { - bicepDeploymentData, err := p.plan(ctx) - if err != nil { - return nil, err - } + var changes []*provisioning.DeploymentPreviewChange - p.console.ShowSpinner(ctx, "Generating infrastructure preview", input.Step) + for _, modulePath := range p.modulePaths() { + bicepDeploymentData, err := p.plan(ctx, modulePath) + if err != nil { + return nil, err + } - targetScope := bicepDeploymentData.Target - deployPreviewResult, err := targetScope.DeployPreview( - ctx, - bicepDeploymentData.CompiledBicep.RawArmTemplate, - bicepDeploymentData.CompiledBicep.Parameters, - ) - if err != nil { - return nil, err - } + p.console.ShowSpinner(ctx, "Generating infrastructure preview", input.Step) - if deployPreviewResult.Error != nil { - deploymentErr := *deployPreviewResult.Error - errDetailsList := make([]string, len(deploymentErr.Details)) - for index, errDetail := range deploymentErr.Details { - errDetailsList[index] = fmt.Sprintf( - "code: %s, message: %s", - convert.ToValueWithDefault(errDetail.Code, ""), - convert.ToValueWithDefault(errDetail.Message, ""), - ) + targetScope := bicepDeploymentData.Target + deployPreviewResult, err := targetScope.DeployPreview( + ctx, + bicepDeploymentData.CompiledBicep.RawArmTemplate, + bicepDeploymentData.CompiledBicep.Parameters, + ) + if err != nil { + return nil, err } - var errDetails string - if len(errDetailsList) > 0 { - errDetails = fmt.Sprintf(" Details: %s", strings.Join(errDetailsList, "\n")) - } - return nil, fmt.Errorf( - "generating preview: error code: %s, message: %s.%s", - convert.ToValueWithDefault(deploymentErr.Code, ""), - convert.ToValueWithDefault(deploymentErr.Message, ""), - errDetails, - ) - } + if deployPreviewResult.Error != nil { + deploymentErr := *deployPreviewResult.Error + errDetailsList := make([]string, len(deploymentErr.Details)) + for index, errDetail := range deploymentErr.Details { + errDetailsList[index] = fmt.Sprintf( + "code: %s, message: %s", + convert.ToValueWithDefault(errDetail.Code, ""), + convert.ToValueWithDefault(errDetail.Message, ""), + ) + } - var changes []*provisioning.DeploymentPreviewChange - for _, change := range deployPreviewResult.Properties.Changes { - resourceAfter := change.After.(map[string]interface{}) + var errDetails string + if len(errDetailsList) > 0 { + errDetails = fmt.Sprintf(" Details: %s", strings.Join(errDetailsList, "\n")) + } + return nil, fmt.Errorf( + "generating preview: error code: %s, message: %s.%s", + convert.ToValueWithDefault(deploymentErr.Code, ""), + convert.ToValueWithDefault(deploymentErr.Message, ""), + errDetails, + ) + } - changes = append(changes, &provisioning.DeploymentPreviewChange{ - ChangeType: provisioning.ChangeType(*change.ChangeType), - ResourceId: provisioning.Resource{ - Id: *change.ResourceID, - }, - ResourceType: resourceAfter["type"].(string), - Name: resourceAfter["name"].(string), - }) + for _, change := range deployPreviewResult.Properties.Changes { + resourceAfter := change.After.(map[string]interface{}) + + if !slices.ContainsFunc(changes, func(c *provisioning.DeploymentPreviewChange) bool { + return c.ChangeType == provisioning.ChangeType(*change.ChangeType) && + c.ResourceId.Id == *change.ResourceID + }) { + changes = append(changes, &provisioning.DeploymentPreviewChange{ + ChangeType: provisioning.ChangeType(*change.ChangeType), + ResourceId: provisioning.Resource{ + Id: *change.ResourceID, + }, + ResourceType: resourceAfter["type"].(string), + Name: resourceAfter["name"].(string), + }) + } + } } return &provisioning.DeployPreviewResult{ Preview: &provisioning.DeploymentPreview{ - Status: *deployPreviewResult.Status, + Status: "done", Properties: &provisioning.DeploymentPreviewProperties{ Changes: changes, }, @@ -731,43 +857,67 @@ func (p *BicepProvider) Destroy( ctx context.Context, options provisioning.DestroyOptions, ) (*provisioning.DestroyResult, error) { - modulePath := p.modulePath() - p.console.ShowSpinner(ctx, "Discovering resources to delete...", input.Step) - defer p.console.StopSpinner(ctx, "", input.StepDone) - compileResult, err := p.compileBicep(ctx, modulePath) - if err != nil { - return nil, fmt.Errorf("creating template: %w", err) - } + modulePaths := p.modulePaths() + rootModulePath := p.rootModulePath() + invalidatedEnvKeys := map[string]bool{} + + deploymentsToDelete := make([]infra.Deployment, 0, len(modulePaths)) + allResourcesToDelete := []*armresources.ResourceReference{} + compileResults := make([]*compileBicepResult, 0, len(modulePaths)) + mostRecentDeployments := make([]*azapi.ResourceDeployment, 0, len(modulePaths)) + scopes := make([]infra.Scope, 0, len(modulePaths)) + for _, modulePath := range modulePaths { + p.console.ShowSpinner(ctx, "Discovering resources to delete...", input.Step) + defer p.console.StopSpinner(ctx, "", input.StepDone) + compileResult, err := p.compileBicep(ctx, modulePath) + if err != nil { + return nil, fmt.Errorf("creating template: %w", err) + } - scope, err := p.scopeForTemplate(compileResult.Template) - if err != nil { - return nil, fmt.Errorf("computing deployment scope: %w", err) - } + scope, err := p.scopeForTemplate(compileResult.Template) + if err != nil { + return nil, fmt.Errorf("computing deployment scope: %w", err) + } - completedDeployments, err := p.deploymentManager.CompletedDeployments(ctx, scope, p.env.Name(), "") - if err != nil { - return nil, fmt.Errorf("finding completed deployments: %w", err) - } + subModuleName := moduleNameFromPaths(modulePath, rootModulePath) + completedDeployments, err := p.deploymentManager.CompletedDeployments(ctx, scope, p.env.Name(), subModuleName, "") + if err != nil { + return nil, fmt.Errorf("finding completed deployments: %w", err) + } - if len(completedDeployments) == 0 { - return nil, fmt.Errorf("no deployments found for environment, '%s'", p.env.Name()) - } + if len(completedDeployments) == 0 { + return nil, fmt.Errorf("no deployments found for environment, '%s'", p.env.Name()) + } - mostRecentDeployment := completedDeployments[0] - deploymentToDelete := scope.Deployment(mostRecentDeployment.Name) + mostRecentDeployment := completedDeployments[0] + deploymentToDelete := scope.Deployment(mostRecentDeployment.Name) - resourcesToDelete, err := deploymentToDelete.Resources(ctx) - if err != nil { - return nil, fmt.Errorf("getting resources to delete: %w", err) + resourcesToDelete, err := deploymentToDelete.Resources(ctx) + if err != nil { + return nil, fmt.Errorf("getting resources to delete: %w", err) + } + + allResourcesToDelete = append(allResourcesToDelete, resourcesToDelete...) + deploymentsToDelete = append(deploymentsToDelete, deploymentToDelete) + compileResults = append(compileResults, compileResult) + mostRecentDeployments = append(mostRecentDeployments, mostRecentDeployment) + scopes = append(scopes, scope) } - groupedResources, err := azapi.GroupByResourceGroup(resourcesToDelete) + groupedResources, err := azapi.GroupByResourceGroup(allResourcesToDelete) if err != nil { return nil, fmt.Errorf("mapping resources to resource groups: %w", err) } if len(groupedResources) == 0 { - return nil, fmt.Errorf("%w, '%s'", infra.ErrDeploymentResourcesNotFound, deploymentToDelete.Name()) + deploymentNames := make([]string, len(deploymentsToDelete)) + for i, deployment := range deploymentsToDelete { + deploymentNames[i] = deployment.Name() + } + return nil, fmt.Errorf( + "%w, '%s'", + infra.ErrDeploymentResourcesNotFound, + strings.Join(deploymentNames, ", ")) } keyVaults, err := p.getKeyVaultsToPurge(ctx, groupedResources) @@ -796,12 +946,13 @@ func (p *BicepProvider) Destroy( } p.console.StopSpinner(ctx, "", input.StepDone) + if err := p.destroyDeploymentWithConfirmation( ctx, options, - deploymentToDelete, + deploymentsToDelete, groupedResources, - len(resourcesToDelete), + len(allResourcesToDelete), ); err != nil { return nil, fmt.Errorf("deleting resource groups: %w", err) } @@ -860,21 +1011,24 @@ func (p *BicepProvider) Destroy( return nil, fmt.Errorf("purging resources: %w", err) } - destroyResult := &provisioning.DestroyResult{ - InvalidatedEnvKeys: slices.Collect(maps.Keys(p.createOutputParameters( - compileResult.Template.Outputs, - azapi.CreateDeploymentOutput(mostRecentDeployment.Outputs), - ))), - } + for i := range deploymentsToDelete { + envKeysInvalidated := p.createOutputParameters( + compileResults[i].Template.Outputs, + azapi.CreateDeploymentOutput(mostRecentDeployments[i].Outputs)) + // Since we have deleted the resource group, add AZURE_RESOURCE_GROUP to the list of invalidated env vars + // so it will be removed from the .env file. + if _, ok := scopes[i].(*infra.ResourceGroupScope); ok { + envKeysInvalidated[environment.ResourceGroupEnvVarName] = provisioning.OutputParameter{} + } - // Since we have deleted the resource group, add AZURE_RESOURCE_GROUP to the list of invalidated env vars - // so it will be removed from the .env file. - if _, ok := scope.(*infra.ResourceGroupScope); ok { - destroyResult.InvalidatedEnvKeys = append( - destroyResult.InvalidatedEnvKeys, environment.ResourceGroupEnvVarName, - ) + for key := range envKeysInvalidated { + invalidatedEnvKeys[key] = true + } } + destroyResult := &provisioning.DestroyResult{ + InvalidatedEnvKeys: slices.Collect(maps.Keys(invalidatedEnvKeys)), + } return destroyResult, nil } @@ -1003,7 +1157,7 @@ func (p *BicepProvider) generateResourcesToDelete(groupedResources map[string][] func (p *BicepProvider) destroyDeploymentWithConfirmation( ctx context.Context, options provisioning.DestroyOptions, - deployment infra.Deployment, + deployments []infra.Deployment, groupedResources map[string][]*azapi.Resource, resourceCount int, ) error { @@ -1041,12 +1195,18 @@ func (p *BicepProvider) destroyDeploymentWithConfirmation( p.console.StopSpinner(ctx, progressMessage.Message, input.StepFailed) } }, func(progress *async.Progress[azapi.DeleteDeploymentProgress]) error { - optionsMap, err := convert.ToMap(p.options) - if err != nil { - return err - } + for _, deployment := range deployments { + optionsMap, err := convert.ToMap(p.options) + if err != nil { + return err + } - return deployment.Delete(ctx, optionsMap, progress) + err = deployment.Delete(ctx, optionsMap, progress) + if err != nil { + return err + } + } + return nil }) if err != nil { @@ -1719,21 +1879,9 @@ func (p *BicepProvider) convertToDeployment(bicepTemplate azure.ArmTemplate) (*p return &template, nil } -// Deploys the specified Bicep module and parameters with the selected provisioning scope (subscription vs resource group) -func (p *BicepProvider) deployModule( - ctx context.Context, - target infra.Deployment, - armTemplate azure.RawArmTemplate, - armParameters azure.ArmParameters, - tags map[string]*string, - options map[string]any, -) (*azapi.ResourceDeployment, error) { - return target.Deploy(ctx, armTemplate, armParameters, tags, options) -} - // Returns either the bicep or bicepparam module file located in the infrastructure root. // The bicepparam file is preferred over bicep file. -func (p *BicepProvider) modulePath() string { +func (p *BicepProvider) rootModulePath() string { infraRoot := p.options.Path moduleName := p.options.Module @@ -1753,6 +1901,41 @@ func (p *BicepProvider) modulePath() string { return filepath.Join(infraRoot, moduleFilename) } +// Returns the path to the module file located in infra/azd, if it exists. +// An empty string is returned if the file does not exist. +func (p *BicepProvider) azdModulePath() string { + infraRoot := p.options.Path + if !filepath.IsAbs(infraRoot) { + infraRoot = filepath.Join(p.projectPath, infraRoot) + } + + moduleDir := filepath.Join(infraRoot, "azd") + moduleFile := filepath.Join(moduleDir, "main.bicepparam") + if _, err := os.Stat(moduleFile); err == nil { + return moduleFile + } + + moduleFile = filepath.Join(moduleDir, "main.bicep") + if _, err := os.Stat(moduleFile); err == nil { + return moduleFile + } + + return "" +} + +// modulePaths returns the paths to all module files available for deployment. +func (p *BicepProvider) modulePaths() []string { + result := []string{ + p.rootModulePath(), + } + + if moduleFile := p.azdModulePath(); moduleFile != "" { + result = append(result, moduleFile) + } + + return result +} + // inputsParameter generates and updates input parameters for the Azure Resource Manager (ARM) template. // It takes an existingInputs map that contains the current input values for each resource, and an autoGenParameters map // that contains information about the input parameters to be generated. @@ -2075,16 +2258,17 @@ func NewBicepProvider( cloud *cloud.Cloud, ) provisioning.Provider { return &BicepProvider{ - envManager: envManager, - env: env, - console: console, - azCli: azCli, - bicepCli: bicepCli, - resourceService: resourceService, - deploymentManager: deploymentManager, - prompters: prompters, - curPrincipal: curPrincipal, - keyvaultService: keyvaultService, - portalUrlBase: cloud.PortalUrlBase, + envManager: envManager, + env: env, + console: console, + azCli: azCli, + bicepCli: bicepCli, + resourceService: resourceService, + deploymentManager: deploymentManager, + prompters: prompters, + curPrincipal: curPrincipal, + keyvaultService: keyvaultService, + portalUrlBase: cloud.PortalUrlBase, + compileBicepMemoryCache: make(map[string]compileBicepResult), } } diff --git a/cli/azd/pkg/infra/provisioning/bicep/bicep_provider_test.go b/cli/azd/pkg/infra/provisioning/bicep/bicep_provider_test.go index a4b74931d72..2394cfb4b59 100644 --- a/cli/azd/pkg/infra/provisioning/bicep/bicep_provider_test.go +++ b/cli/azd/pkg/infra/provisioning/bicep/bicep_provider_test.go @@ -48,7 +48,7 @@ func TestBicepPlan(t *testing.T) { prepareBicepMocks(mockContext) infraProvider := createBicepProvider(t, mockContext) - deploymentPlan, err := infraProvider.plan(*mockContext.Context) + deploymentPlan, err := infraProvider.plan(*mockContext.Context, infraProvider.rootModulePath()) require.Nil(t, err) @@ -104,7 +104,7 @@ func TestBicepPlanPrompt(t *testing.T) { }).Respond(false) infraProvider := createBicepProvider(t, mockContext) - plan, err := infraProvider.plan(*mockContext.Context) + plan, err := infraProvider.plan(*mockContext.Context, infraProvider.rootModulePath()) require.NoError(t, err) @@ -290,7 +290,7 @@ func TestPlanForResourceGroup(t *testing.T) { infraProvider := createBicepProvider(t, mockContext) // The computed plan should target the resource group we picked. - planResult, err := infraProvider.plan(*mockContext.Context) + planResult, err := infraProvider.plan(*mockContext.Context, infraProvider.rootModulePath()) require.Nil(t, err) require.NotNil(t, planResult) require.Equal(t, "rg-test-env", @@ -868,7 +868,7 @@ func TestFindCompletedDeployments(t *testing.T) { *mockContext.Context, &mockedScope{ baseDate: baseDate, envTag: envTag, - }, envTag, "") + }, envTag, "", "") require.NoError(t, err) require.Equal(t, 1, len(deployments)) // should take the base date + 2 years diff --git a/cli/azd/pkg/infra/provisioning_progress_display.go b/cli/azd/pkg/infra/provisioning_progress_display.go index 32727442e12..eff6d66bd8f 100644 --- a/cli/azd/pkg/infra/provisioning_progress_display.go +++ b/cli/azd/pkg/infra/provisioning_progress_display.go @@ -7,9 +7,7 @@ import ( "context" "fmt" "log" - "os" "sort" - "strconv" "strings" "time" @@ -23,71 +21,93 @@ import ( // ProvisioningProgressDisplay displays interactive progress for an ongoing Azure provisioning operation. type ProvisioningProgressDisplay struct { + // demo mode, controls whether links to Azure Portal are displayed + demoMode bool // Whether the deployment has started - deploymentStarted bool + deploymentDisplayed map[string]bool + // Total number of deployments + totalDeployments int // Keeps track of created resources displayedResources map[string]bool resourceManager ResourceManager console input.Console - deployment Deployment } func NewProvisioningProgressDisplay( rm ResourceManager, console input.Console, - deployment Deployment, + demoMode bool, + totalDeployments int, ) *ProvisioningProgressDisplay { return &ProvisioningProgressDisplay{ - displayedResources: map[string]bool{}, - deployment: deployment, - resourceManager: rm, - console: console, + displayedResources: map[string]bool{}, + deploymentDisplayed: map[string]bool{}, + resourceManager: rm, + console: console, + demoMode: demoMode, + totalDeployments: totalDeployments, } } // ReportProgress reports the current deployment progress, setting the currently executing operation title and logging // progress. -func (display *ProvisioningProgressDisplay) ReportProgress( - ctx context.Context, queryStart *time.Time) error { - if !display.deploymentStarted { - _, err := display.deployment.Get(ctx) - if err != nil { - // Return default progress - log.Printf("error while reporting progress: %s", err.Error()) - return nil +func (d *ProvisioningProgressDisplay) ReportProgress( + ctx context.Context, deployment Deployment, queryStart *time.Time) error { + name := deployment.Name() + if d.demoMode && !d.deploymentDisplayed[name] { + lines := []string{ + "You can view detailed progress in the Azure Portal:", + "\n", } - display.deploymentStarted = true - deploymentUrl, err := display.deployment.DeploymentUrl(ctx) + if d.totalDeployments > 1 { + lines = []string{ + fmt.Sprintf("Deployment %d of %d:", len(d.deploymentDisplayed)+1, d.totalDeployments), + "\n", + } + } + + d.console.EnsureBlankLine(ctx) + d.console.MessageUxItem( + ctx, + &ux.MultilineMessage{ + Lines: lines, + }, + ) + d.deploymentDisplayed[name] = true + } else if !d.deploymentDisplayed[name] { + deploymentUrl, err := deployment.DeploymentUrl(ctx) if err != nil { - return err + // Wait until deployment is live to display progress + return nil } deploymentLink := fmt.Sprintf(output.WithLinkFormat("%s\n"), deploymentUrl) - - display.console.EnsureBlankLine(ctx) - + d.console.EnsureBlankLine(ctx) lines := []string{ "You can view detailed progress in the Azure Portal:", deploymentLink, } - if v, err := strconv.ParseBool(os.Getenv("AZD_DEMO_MODE")); err == nil && v { + if d.totalDeployments > 1 { lines = []string{ - "You can view detailed progress in the Azure Portal.", - "\n", + fmt.Sprintf( + "Deployment %d of %d. View detailed progress in the Azure Portal:", + len(d.deploymentDisplayed)+1, d.totalDeployments), + deploymentLink, } } - - display.console.MessageUxItem( + d.console.MessageUxItem( ctx, &ux.MultilineMessage{ Lines: lines, }, ) + + d.deploymentDisplayed[name] = true } - operations, err := display.resourceManager.GetDeploymentResourceOperations(ctx, display.deployment, queryStart) + operations, err := d.resourceManager.GetDeploymentResourceOperations(ctx, deployment, queryStart) if err != nil { // Status display is best-effort activity. return err @@ -101,7 +121,7 @@ func (display *ProvisioningProgressDisplay) ReportProgress( if operations[i].Properties.TargetResource != nil { resourceId := *operations[i].Properties.TargetResource.ResourceName - if !display.displayedResources[resourceId] { + if !d.displayedResources[resourceId] { switch *operations[i].Properties.ProvisioningState { case string(armresources.ProvisioningStateSucceeded): newlyDeployedResources = append(newlyDeployedResources, operations[i]) @@ -122,20 +142,21 @@ func (display *ProvisioningProgressDisplay) ReportProgress( }) displayedResources := append(newlyDeployedResources, newlyFailedResources...) - display.logNewlyCreatedResources(ctx, displayedResources, runningDeployments) + d.logNewlyCreatedResources(ctx, displayedResources, deployment, runningDeployments) return nil } func (display *ProvisioningProgressDisplay) logNewlyCreatedResources( ctx context.Context, resources []*armresources.DeploymentOperation, + deployment Deployment, inProgressResources []*armresources.DeploymentOperation, ) { for _, resource := range resources { resourceTypeName := *resource.Properties.TargetResource.ResourceType resourceTypeDisplayName, err := display.resourceManager.GetResourceTypeDisplayName( ctx, - display.deployment.SubscriptionId(), + deployment.SubscriptionId(), *resource.Properties.TargetResource.ID, azapi.AzureResourceType(resourceTypeName), ) @@ -180,7 +201,7 @@ func (display *ProvisioningProgressDisplay) logNewlyCreatedResources( resourceTypeName := *inProgResource.Properties.TargetResource.ResourceType resourceTypeDisplayName, err := display.resourceManager.GetResourceTypeDisplayName( ctx, - display.deployment.SubscriptionId(), + deployment.SubscriptionId(), *inProgResource.Properties.TargetResource.ID, azapi.AzureResourceType(resourceTypeName), ) diff --git a/cli/azd/pkg/infra/provisioning_progress_display_test.go b/cli/azd/pkg/infra/provisioning_progress_display_test.go index f097210eeab..5bda703b5c9 100644 --- a/cli/azd/pkg/infra/provisioning_progress_display_test.go +++ b/cli/azd/pkg/infra/provisioning_progress_display_test.go @@ -139,8 +139,8 @@ func TestReportProgress(t *testing.T) { startTime := time.Now() outputLength := 0 mockResourceManager := mockResourceManager{} - progressDisplay := NewProvisioningProgressDisplay(&mockResourceManager, mockContext.Console, deployment) - err := progressDisplay.ReportProgress(*mockContext.Context, &startTime) + progressDisplay := NewProvisioningProgressDisplay(&mockResourceManager, mockContext.Console, false, 1) + err := progressDisplay.ReportProgress(*mockContext.Context, deployment, &startTime) require.NoError(t, err) outputLength++ @@ -148,7 +148,7 @@ func TestReportProgress(t *testing.T) { assert.Contains(t, mockContext.Console.Output()[0], "You can view detailed progress in the Azure Portal:") mockResourceManager.AddInProgressOperation() - err = progressDisplay.ReportProgress(*mockContext.Context, &startTime) + err = progressDisplay.ReportProgress(*mockContext.Context, deployment, &startTime) require.NoError(t, err) assert.Len(t, mockContext.Console.Output(), outputLength) } From 40071c147b88e3f1c11cf37b1798b730f484496b Mon Sep 17 00:00:00 2001 From: Wei Lim Date: Mon, 11 Nov 2024 15:42:30 -0800 Subject: [PATCH 3/3] compose if infra exists --- cli/azd/pkg/project/importer.go | 70 +++++++++++++++++++++++----- cli/azd/pkg/project/importer_test.go | 3 +- cli/azd/pkg/project/scaffold_gen.go | 66 +++++++++++++++----------- 3 files changed, 99 insertions(+), 40 deletions(-) diff --git a/cli/azd/pkg/project/importer.go b/cli/azd/pkg/project/importer.go index 26fbde3a07e..384ed1e32ca 100644 --- a/cli/azd/pkg/project/importer.go +++ b/cli/azd/pkg/project/importer.go @@ -5,6 +5,7 @@ package project import ( "context" + "errors" "fmt" "io/fs" "log" @@ -15,6 +16,7 @@ import ( "github.com/azure/azure-dev/cli/azd/pkg/alpha" "github.com/azure/azure-dev/cli/azd/pkg/infra/provisioning" + "github.com/otiai10/copy" ) type ImportManager struct { @@ -139,8 +141,63 @@ func (im *ImportManager) ProjectInfrastructure(ctx context.Context, projectConfi infraRoot = filepath.Join(projectConfig.Path, infraRoot) } + moduleExists, moduleErr := pathHasModule(infraRoot, projectConfig.Infra.Module) + + composeEnabled := im.dotNetImporter.alphaFeatureManager.IsEnabled(featureCompose) + if composeEnabled && len(projectConfig.Resources) > 0 { + if moduleErr == nil && moduleExists { + azdModuleExists, err := pathHasModule(filepath.Join(infraRoot, "azd"), projectConfig.Infra.Module) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("checking if module exists: %w", err) + } + + if azdModuleExists { + log.Printf("using fully-synthesized infrastructure from %s directory", infraRoot) + return &Infra{ + Options: projectConfig.Infra, + }, nil + } + } + + // copy the infra directory to a temporary directory and synthesize the azd directory + tmpDir, err := os.MkdirTemp("", "azd-infra") + if err != nil { + return nil, fmt.Errorf("creating temporary directory: %w", err) + } + + azdInfraDir := tmpDir + if moduleErr == nil && moduleExists { + // Copy the base infra directory + if err := copy.Copy(infraRoot, tmpDir); err != nil { + return nil, fmt.Errorf("copying infra directory: %w", err) + } + + azdInfraDir = filepath.Join(tmpDir, "azd") + } + + err = infraFsToDir(ctx, projectConfig, azdInfraDir) + if err != nil { + return nil, err + } + + return &Infra{ + Options: provisioning.Options{ + Provider: provisioning.Bicep, + Path: tmpDir, + Module: DefaultModule, + }, + cleanupDir: tmpDir, + }, nil + } + + if !composeEnabled && len(projectConfig.Resources) > 0 { + return nil, fmt.Errorf( + "compose is currently under alpha support and must be explicitly enabled."+ + " Run `%s` to enable this feature", alpha.GetEnableCommand(featureCompose)) + } + // Allow overriding the infrastructure only when path and module exists. - if moduleExists, err := pathHasModule(infraRoot, projectConfig.Infra.Module); err == nil && moduleExists { + if moduleErr == nil && moduleExists { log.Printf("using infrastructure from %s directory", infraRoot) return &Infra{ Options: projectConfig.Infra, @@ -165,17 +222,6 @@ func (im *ImportManager) ProjectInfrastructure(ctx context.Context, projectConfi } } - composeEnabled := im.dotNetImporter.alphaFeatureManager.IsEnabled(featureCompose) - if composeEnabled && len(projectConfig.Resources) > 0 { - return tempInfra(ctx, projectConfig) - } - - if !composeEnabled && len(projectConfig.Resources) > 0 { - return nil, fmt.Errorf( - "compose is currently under alpha support and must be explicitly enabled."+ - " Run `%s` to enable this feature", alpha.GetEnableCommand(featureCompose)) - } - return &Infra{}, nil } diff --git a/cli/azd/pkg/project/importer_test.go b/cli/azd/pkg/project/importer_test.go index 168e5c93261..594da660006 100644 --- a/cli/azd/pkg/project/importer_test.go +++ b/cli/azd/pkg/project/importer_test.go @@ -233,7 +233,8 @@ func TestImportManagerProjectInfrastructure(t *testing.T) { lazyEnvManager: lazy.NewLazy(func() (environment.Manager, error) { return mockEnv, nil }), - hostCheck: make(map[string]hostCheckResult), + hostCheck: make(map[string]hostCheckResult), + alphaFeatureManager: mockContext.AlphaFeaturesManager, }) // Do not use defaults diff --git a/cli/azd/pkg/project/scaffold_gen.go b/cli/azd/pkg/project/scaffold_gen.go index 120f1c63211..1ce8436ab8c 100644 --- a/cli/azd/pkg/project/scaffold_gen.go +++ b/cli/azd/pkg/project/scaffold_gen.go @@ -5,6 +5,7 @@ package project import ( "context" + "errors" "fmt" "io/fs" "os" @@ -13,7 +14,6 @@ import ( "strings" "github.com/azure/azure-dev/cli/azd/internal/scaffold" - "github.com/azure/azure-dev/cli/azd/pkg/infra/provisioning" "github.com/azure/azure-dev/cli/azd/pkg/osutil" "github.com/psanford/memfs" ) @@ -38,18 +38,10 @@ func infraFs(_ context.Context, prjConfig *ProjectConfig) (fs.FS, error) { return files, nil } -// Returns the infrastructure configuration that points to a temporary, generated `infra` directory on the filesystem. -func tempInfra( - ctx context.Context, - prjConfig *ProjectConfig) (*Infra, error) { - tmpDir, err := os.MkdirTemp("", "azd-infra") - if err != nil { - return nil, fmt.Errorf("creating temporary directory: %w", err) - } - +func infraFsToDir(ctx context.Context, prjConfig *ProjectConfig, dir string) error { files, err := infraFs(ctx, prjConfig) if err != nil { - return nil, err + return err } err = fs.WalkDir(files, ".", func(path string, d fs.DirEntry, err error) error { @@ -61,7 +53,7 @@ func tempInfra( return nil } - target := filepath.Join(tmpDir, path) + target := filepath.Join(dir, path) if err := os.MkdirAll(filepath.Dir(target), osutil.PermissionDirectoryOwnerOnly); err != nil { return err } @@ -74,17 +66,10 @@ func tempInfra( return os.WriteFile(target, contents, d.Type().Perm()) }) if err != nil { - return nil, fmt.Errorf("writing infrastructure: %w", err) + return fmt.Errorf("writing infrastructure: %w", err) } - return &Infra{ - Options: provisioning.Options{ - Provider: provisioning.Bicep, - Path: tmpDir, - Module: DefaultModule, - }, - cleanupDir: tmpDir, - }, nil + return nil } // Generates the filesystem of all infrastructure files to be placed, rooted at the project directory. @@ -95,13 +80,32 @@ func infraFsForProject(ctx context.Context, prjConfig *ProjectConfig) (fs.FS, er return nil, err } - infraPathPrefix := DefaultPath + infraPrefix := DefaultPath if prjConfig.Infra.Path != "" { - infraPathPrefix = prjConfig.Infra.Path + infraPrefix = prjConfig.Infra.Path + } + + infraRoot := infraPrefix + if !filepath.IsAbs(infraPrefix) { + infraRoot = filepath.Join(prjConfig.Path, infraPrefix) + } + + infraDir, err := os.Stat(infraRoot) + if !errors.Is(err, os.ErrNotExist) && err != nil { + return nil, fmt.Errorf("error reading infra directory: %w", err) + } + + fi, err := os.Stat(filepath.Join(infraRoot, ".azd")) + if !errors.Is(err, os.ErrNotExist) && err != nil { + return nil, fmt.Errorf("error reading .azd file in infra: %w", err) + } + + if infraDir != nil && fi == nil { // if the infra directory is not managed by azd, generate it to infra/azd + infraPrefix = filepath.Join(infraPrefix, "azd") } - // root the generated content at the project directory generatedFS := memfs.New() + // root the generated content at the project directory err = fs.WalkDir(infraFS, ".", func(path string, d fs.DirEntry, err error) error { if err != nil { return err @@ -111,7 +115,7 @@ func infraFsForProject(ctx context.Context, prjConfig *ProjectConfig) (fs.FS, er return nil } - err = generatedFS.MkdirAll(filepath.Join(infraPathPrefix, filepath.Dir(path)), osutil.PermissionDirectoryOwnerOnly) + err = generatedFS.MkdirAll(filepath.Join(infraPrefix, filepath.Dir(path)), osutil.PermissionDirectoryOwnerOnly) if err != nil { return err } @@ -121,10 +125,18 @@ func infraFsForProject(ctx context.Context, prjConfig *ProjectConfig) (fs.FS, er return err } - return generatedFS.WriteFile(filepath.Join(infraPathPrefix, path), contents, d.Type().Perm()) + return generatedFS.WriteFile(filepath.Join(infraPrefix, path), contents, d.Type().Perm()) }) if err != nil { - return nil, err + return nil, fmt.Errorf("generating: %w", err) + } + + if fi == nil { + // create a sentinel file to indicate that the infra directory is managed by azd + err = generatedFS.WriteFile(filepath.Join(infraPrefix, ".azd"), []byte{}, osutil.PermissionFileOwnerOnly) + if err != nil { + return nil, fmt.Errorf("writing sentinel: %w", err) + } } return generatedFS, nil