From ddf75967cd613c7190d005afc97bcb3ea8048bfe Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Thu, 19 Dec 2024 14:18:18 +0000 Subject: [PATCH 1/2] EES-XXXX - added ability to specific workload profiles and container resource limits for Container Apps --- .../application/public-api/publicApiApp.bicep | 9 +++- .../shared/containerAppEnvironment.bicep | 8 +++- .../public-api/components/containerApp.bicep | 47 ++++++++++--------- .../components/containerAppEnvironment.bicep | 19 ++++---- .../templates/public-api/main.bicep | 16 ++++++- .../public-api/parameters/main-dev.bicepparam | 8 ++++ .../templates/public-api/types.bicep | 17 +++++++ 7 files changed, 89 insertions(+), 35 deletions(-) diff --git a/infrastructure/templates/public-api/application/public-api/publicApiApp.bicep b/infrastructure/templates/public-api/application/public-api/publicApiApp.bicep index 414920edf6e..04edeef331b 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiApp.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiApp.bicep @@ -1,4 +1,4 @@ -import { ResourceNames } from '../../types.bicep' +import { ResourceNames, ContainerAppResourceConfig } from '../../types.bicep' @description('Specifies common resource naming variables.') param resourceNames ResourceNames @@ -30,6 +30,9 @@ param apiAppRegistrationClientId string @description('Specifies the Application Insights connection string for this Container App to use for its monitoring.') param appInsightsConnectionString string +@description('Resource limits and scaling configuration.') +param resourceAndScalingConfig ContainerAppResourceConfig + @description('Whether to create or update Azure Monitor alerts during this deploy') param deployAlerts bool @@ -145,6 +148,10 @@ module apiContainerAppModule '../../components/containerApp.bicep' = { ] requireAuthentication: false } + cpuCores: resourceAndScalingConfig.cpuCores + memorySizeGis: resourceAndScalingConfig.memoryGis + minReplicas: resourceAndScalingConfig.minReplicas + maxReplicas: resourceAndScalingConfig.maxReplicas tagValues: tagValues } } diff --git a/infrastructure/templates/public-api/application/shared/containerAppEnvironment.bicep b/infrastructure/templates/public-api/application/shared/containerAppEnvironment.bicep index 6222b2ea4cf..4be0593da07 100644 --- a/infrastructure/templates/public-api/application/shared/containerAppEnvironment.bicep +++ b/infrastructure/templates/public-api/application/shared/containerAppEnvironment.bicep @@ -1,4 +1,4 @@ -import { ResourceNames } from '../../types.bicep' +import { ResourceNames, ContainerAppWorkloadProfile } from '../../types.bicep' @description('Specifies common resource naming variables.') param resourceNames ResourceNames @@ -9,6 +9,9 @@ param location string @description('Specifies the Application Insights key that is associated with this resource.') param applicationInsightsKey string +@description('Specifies the workload profiles for this Container App Environment - the default Consumption plan is always included') +param workloadProfiles ContainerAppWorkloadProfile[] = [] + @description('Specifies a set of tags with which to tag the resource in Azure.') param tagValues object @@ -33,7 +36,6 @@ module containerAppEnvironmentModule '../../components/containerAppEnvironment.b subnetId: subnet.id logAnalyticsWorkspaceName: resourceNames.sharedResources.logAnalyticsWorkspace applicationInsightsKey: applicationInsightsKey - tagValues: tagValues azureFileStorages: [ { storageName: resourceNames.publicApi.publicApiFileShare @@ -43,6 +45,8 @@ module containerAppEnvironmentModule '../../components/containerAppEnvironment.b accessMode: 'ReadWrite' } ] + workloadProfiles: workloadProfiles + tagValues: tagValues } } diff --git a/infrastructure/templates/public-api/components/containerApp.bicep b/infrastructure/templates/public-api/components/containerApp.bicep index cdb8f6804bf..24ad3fe1562 100644 --- a/infrastructure/templates/public-api/components/containerApp.bicep +++ b/infrastructure/templates/public-api/components/containerApp.bicep @@ -25,37 +25,40 @@ param corsPolicy { allowedOrigins: string[]? } +@description('Name of the workload profile under which this Container App will be deployed. Defaults to Consumption.') +param workloadProfileName string = 'Consumption' + @description('Number of CPU cores the container can use. Can be with a maximum of two decimals.') @allowed([ - '1' - '2' - '3' - '4' + 1 + 2 + 3 + 4 ]) -param cpuCore string = '4' +param cpuCores int = 4 @description('Amount of memory (in gibibytes, GiB) allocated to the container up to 4GiB. Can be with a maximum of two decimals. Ratio with CPU cores must be equal to 2.') @allowed([ - '1' - '2' - '3' - '4' - '5' - '6' - '7' - '8' + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 ]) -param memorySize string = '8' +param memorySizeGis int = 8 @description('Minimum number of replicas that will be deployed') @minValue(0) @maxValue(25) -param minReplica int = 1 +param minReplicas int = 1 @description('Maximum number of replicas that will be deployed') @minValue(0) @maxValue(25) -param maxReplica int = 3 +param maxReplicas int = 3 @description('Specifies the database connection string') param appSettings { @@ -113,7 +116,7 @@ param tagValues object var containerImageName = '${acrLoginServer}/${containerAppImageName}' -resource containerApp 'Microsoft.App/containerApps@2023-11-02-preview' = { +resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { name: containerAppName location: location identity: { @@ -160,15 +163,15 @@ resource containerApp 'Microsoft.App/containerApps@2023-11-02-preview' = { image: containerImageName env: appSettings resources: { - cpu: json(cpuCore) - memory: '${memorySize}Gi' + cpu: json(string(cpuCores)) + memory: '${memorySizeGis}Gi' } volumeMounts: volumeMounts } ] scale: { - minReplicas: minReplica - maxReplicas: maxReplica + minReplicas: minReplicas + maxReplicas: maxReplicas rules: [ { name: 'http-requests' @@ -182,7 +185,7 @@ resource containerApp 'Microsoft.App/containerApps@2023-11-02-preview' = { } volumes: volumes } - workloadProfileName: 'Consumption' + workloadProfileName: workloadProfileName } tags: tagValues } diff --git a/infrastructure/templates/public-api/components/containerAppEnvironment.bicep b/infrastructure/templates/public-api/components/containerAppEnvironment.bicep index 93442b6b20a..39436e24765 100644 --- a/infrastructure/templates/public-api/components/containerAppEnvironment.bicep +++ b/infrastructure/templates/public-api/components/containerAppEnvironment.bicep @@ -1,3 +1,5 @@ +import { ContainerAppWorkloadProfile } from '../types.bicep' + @description('Specifies the location of the Container App Environment - defaults to that of the Resource Group') param location string @@ -13,14 +15,8 @@ param logAnalyticsWorkspaceName string @description('Specifies the Application Insights key that is associated with this resource') param applicationInsightsKey string -@description('Specifies the workload profiles for this Container App Environment - defaults to Consumption') -param workloadProfiles { - name: string - workloadProfileType: string -}[] = [{ - name: 'Consumption' - workloadProfileType: 'Consumption' -}] +@description('Specifies the workload profiles for this Container App Environment - the default Consumption plan is always included') +param workloadProfiles ContainerAppWorkloadProfile[] = [] @description('Specifies a set of tags with which to tag the resource in Azure') param tagValues object @@ -57,7 +53,12 @@ resource containerAppEnvironment 'Microsoft.App/managedEnvironments@2024-03-01' sharedKey: logAnalyticsWorkspace.listKeys().primarySharedKey } } - workloadProfiles: workloadProfiles + workloadProfiles: union([{ + name: 'Consumption' + workloadProfileType: 'Consumption' + }], + workloadProfiles + ) } tags: tagValues diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 4426a92c6ac..188456f9e8a 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -1,5 +1,5 @@ import { abbreviations } from 'abbreviations.bicep' -import { IpRange, PrincipalNameAndId, StaticWebAppSku } from 'types.bicep' +import { ContainerAppResourceConfig, IpRange, PrincipalNameAndId, StaticWebAppSku, ContainerAppWorkloadProfile } from 'types.bicep' @description('Environment : Subscription name e.g. s101d01. Used as a prefix for created resources.') param subscription string = '' @@ -107,6 +107,18 @@ param devopsServicePrincipalId string = '' @description('Specifies whether or not test Themes can be deleted in the environment.') param enableThemeDeletion bool = false +@description('Specifies the workload profiles for this Container App Environment - the default Consumption plan is always included') +param publicApiContainerAppWorkloadProfiles ContainerAppWorkloadProfile[] = [] + +@description('Resource configuration for the Public API Container App.') +param publicApiContainerAppConfig ContainerAppResourceConfig = { + cpuCores: 4 + memoryGis: 8 + minReplicas: 0 + maxReplicas: 3 + workloadProfileName: 'Consumption' +} + var tagValues = union(resourceTags ?? {}, { Environment: environmentName DateProvisioned: dateProvisioned @@ -254,6 +266,7 @@ module containerAppEnvironmentModule 'application/shared/containerAppEnvironment location: location resourceNames: resourceNames applicationInsightsKey: appInsightsModule.outputs.appInsightsKey + workloadProfiles: publicApiContainerAppWorkloadProfiles tagValues: tagValues } dependsOn: [ @@ -286,6 +299,7 @@ module apiAppModule 'application/public-api/publicApiApp.bicep' = if (deployCont dockerImagesTag: dockerImagesTag appInsightsConnectionString: appInsightsModule.outputs.appInsightsConnectionString deployAlerts: deployAlerts + resourceAndScalingConfig: publicApiContainerAppConfig tagValues: tagValues } dependsOn: [ diff --git a/infrastructure/templates/public-api/parameters/main-dev.bicepparam b/infrastructure/templates/public-api/parameters/main-dev.bicepparam index 2b7fd120b21..d9b2425967e 100644 --- a/infrastructure/templates/public-api/parameters/main-dev.bicepparam +++ b/infrastructure/templates/public-api/parameters/main-dev.bicepparam @@ -14,4 +14,12 @@ param postgreSqlSkuName = 'Standard_B1ms' param postgreSqlStorageSizeGB = 32 param postgreSqlAutoGrowStatus = 'Disabled' +param publicApiContainerAppConfig = { + cpuCores: 4 + memoryGis: 8 + minReplicas: 1 + maxReplicas: 16 + workloadProfileName: 'Consumption' +} + param enableThemeDeletion = true diff --git a/infrastructure/templates/public-api/types.bicep b/infrastructure/templates/public-api/types.bicep index d21d724fad2..44ea8544cfa 100644 --- a/infrastructure/templates/public-api/types.bicep +++ b/infrastructure/templates/public-api/types.bicep @@ -222,3 +222,20 @@ type KeyVaultRole = 'Secrets User' | 'Certificate User' @export() type StaticWebAppSku = 'Free' | 'Standard' + +@export() +type ContainerAppResourceConfig = { + workloadProfileName: string + minReplicas: int + maxReplicas: int + cpuCores: int + memoryGis: int +} + +@export() +type ContainerAppWorkloadProfile = { + name: string + workloadProfileType: 'D4' | 'D8' | 'D16' | 'D32' | 'E4' | 'E8' | 'E16' | 'E32' + minimumCount: int + maximumCount: int +} From 3ddcaffebf38717c4907500fc785b2fae3b78143 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 20 Dec 2024 11:57:49 +0000 Subject: [PATCH 2/2] EES-XXXX - adding correct restrictions to container limits. Adding default profiles for environments. --- .../application/public-api/publicApiApp.bicep | 4 +- .../public-api/ci/azure-pipelines.yml | 5 +++ .../public-api/ci/jobs/deploy-api-docs.yml | 1 + .../ci/jobs/deploy-infrastructure.yml | 2 + .../public-api/ci/tasks/deploy-bicep.yml | 3 ++ .../public-api/components/containerApp.bicep | 39 +++++++------------ .../templates/public-api/main.bicep | 8 +++- .../public-api/parameters/main-dev.bicepparam | 10 ++++- .../templates/public-api/types.bicep | 5 ++- 9 files changed, 47 insertions(+), 30 deletions(-) diff --git a/infrastructure/templates/public-api/application/public-api/publicApiApp.bicep b/infrastructure/templates/public-api/application/public-api/publicApiApp.bicep index 04edeef331b..2b62ca27c18 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiApp.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiApp.bicep @@ -149,9 +149,11 @@ module apiContainerAppModule '../../components/containerApp.bicep' = { requireAuthentication: false } cpuCores: resourceAndScalingConfig.cpuCores - memorySizeGis: resourceAndScalingConfig.memoryGis + memoryGis: resourceAndScalingConfig.memoryGis minReplicas: resourceAndScalingConfig.minReplicas maxReplicas: resourceAndScalingConfig.maxReplicas + scaleAtConcurrentHttpRequests: resourceAndScalingConfig.scaleAtConcurrentHttpRequests + workloadProfileName: resourceAndScalingConfig.workloadProfileName tagValues: tagValues } } diff --git a/infrastructure/templates/public-api/ci/azure-pipelines.yml b/infrastructure/templates/public-api/ci/azure-pipelines.yml index 5412ea98168..a90d6cc9c57 100644 --- a/infrastructure/templates/public-api/ci/azure-pipelines.yml +++ b/infrastructure/templates/public-api/ci/azure-pipelines.yml @@ -13,6 +13,9 @@ parameters: - name: deployDataProcessor displayName: Does the Data Processor need creating or updating? default: true + - name: deployDocsSite + displayName: Does the Public API static docs site need creating or updating? + default: true - name: deployAlerts displayName: Whether to create or update Azure Monitor alerts during this deploy. default: false @@ -58,6 +61,8 @@ variables: value: ${{ parameters.deployContainerApp }} - name: deployDataProcessor value: ${{ parameters.deployDataProcessor }} + - name: deployDocsSite + value: ${{ parameters.deployDocsSite }} - name: deployAlerts value: ${{ parameters.deployAlerts }} - name: awaitActiveOrchestrations diff --git a/infrastructure/templates/public-api/ci/jobs/deploy-api-docs.yml b/infrastructure/templates/public-api/ci/jobs/deploy-api-docs.yml index a7e17c543f8..1fe51fc720a 100644 --- a/infrastructure/templates/public-api/ci/jobs/deploy-api-docs.yml +++ b/infrastructure/templates/public-api/ci/jobs/deploy-api-docs.yml @@ -10,6 +10,7 @@ parameters: jobs: - deployment: DeployPublicApiDocs displayName: Deploy Public API docs + condition: and(succeeded(), eq(variables.deployDocsSite, true)) dependsOn: ${{ parameters.dependsOn }} environment: ${{ parameters.environment }} strategy: diff --git a/infrastructure/templates/public-api/ci/jobs/deploy-infrastructure.yml b/infrastructure/templates/public-api/ci/jobs/deploy-infrastructure.yml index af0c39481de..056be35954f 100644 --- a/infrastructure/templates/public-api/ci/jobs/deploy-infrastructure.yml +++ b/infrastructure/templates/public-api/ci/jobs/deploy-infrastructure.yml @@ -39,6 +39,7 @@ jobs: deployPsqlFlexibleServer: false deployContainerApp: false deployDataProcessor: false + deployDocsSite: false deployAlerts: false dataProcessorExists: false @@ -68,6 +69,7 @@ jobs: deployPsqlFlexibleServer: $(deployPsqlFlexibleServer) deployContainerApp: $(deployContainerApp) deployDataProcessor: $(deployDataProcessor) + deployDocsSite: $(deployDocsSite) deployAlerts: $(deployAlerts) dataProcessorExists: $(dataProcessorExists) diff --git a/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml b/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml index 2e9d5ca3518..39a0196c10a 100644 --- a/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml +++ b/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml @@ -20,6 +20,8 @@ parameters: default: true - name: deployDataProcessor type: string + - name: deployDocsSite + type: string - name: deployAlerts type: string - name: dataProcessorExists @@ -55,6 +57,7 @@ steps: deployPsqlFlexibleServer=${{ parameters.deployPsqlFlexibleServer }} \ deployContainerApp=${{ parameters.deployContainerApp }} \ deployDataProcessor=${{ parameters.deployDataProcessor }} \ + deployDocsSite=${{ parameters.deployDocsSite }} \ deployAlerts=${{ parameters.deployAlerts }} \ dataProcessorFunctionAppExists=${{ parameters.dataProcessorExists }} \ dataProcessorAppRegistrationClientId='$(dataProcessorAppRegistrationClientId)' \ diff --git a/infrastructure/templates/public-api/components/containerApp.bicep b/infrastructure/templates/public-api/components/containerApp.bicep index 24ad3fe1562..331ef0c0133 100644 --- a/infrastructure/templates/public-api/components/containerApp.bicep +++ b/infrastructure/templates/public-api/components/containerApp.bicep @@ -29,37 +29,28 @@ param corsPolicy { param workloadProfileName string = 'Consumption' @description('Number of CPU cores the container can use. Can be with a maximum of two decimals.') -@allowed([ - 1 - 2 - 3 - 4 -]) +@minValue(1) +@maxValue(8) param cpuCores int = 4 -@description('Amount of memory (in gibibytes, GiB) allocated to the container up to 4GiB. Can be with a maximum of two decimals. Ratio with CPU cores must be equal to 2.') -@allowed([ - 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 -]) -param memorySizeGis int = 8 +@description('Amount of memory (in gibibytes, GiB) allocated to the container up to 32GiB. Can be with a maximum of two decimals. Ratio with CPU cores must be equal to 2.') +@minValue(1) +@maxValue(32) +param memoryGis int = 8 @description('Minimum number of replicas that will be deployed') @minValue(0) -@maxValue(25) param minReplicas int = 1 @description('Maximum number of replicas that will be deployed') @minValue(0) -@maxValue(25) +@maxValue(1000) param maxReplicas int = 3 +@description('Number of concurrent requests required in order to trigger scaling out.') +@minValue(1) +param scaleAtConcurrentHttpRequests int? + @description('Specifies the database connection string') param appSettings { name: string @@ -164,7 +155,7 @@ resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { env: appSettings resources: { cpu: json(string(cpuCores)) - memory: '${memorySizeGis}Gi' + memory: '${memoryGis}Gi' } volumeMounts: volumeMounts } @@ -172,16 +163,16 @@ resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { scale: { minReplicas: minReplicas maxReplicas: maxReplicas - rules: [ + rules: scaleAtConcurrentHttpRequests != null ? [ { name: 'http-requests' http: { metadata: { - concurrentRequests: '10' + concurrentRequests: string(scaleAtConcurrentHttpRequests) } } } - ] + ] : [] } volumes: volumes } diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 188456f9e8a..4fe68b7a649 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -78,6 +78,9 @@ param deployContainerApp bool = true @description('Does the Data Processor need creating or updating?') param deployDataProcessor bool = true +@description('Does the Public API static docs site need creating or updating?') +param deployDocsSite bool = true + param deployAlerts bool = false @description('Public URLs of other components in the service.') @@ -116,6 +119,7 @@ param publicApiContainerAppConfig ContainerAppResourceConfig = { memoryGis: 8 minReplicas: 0 maxReplicas: 3 + scaleAtConcurrentHttpRequests: 10 workloadProfileName: 'Consumption' } @@ -309,7 +313,7 @@ module apiAppModule 'application/public-api/publicApiApp.bicep' = if (deployCont } // Deploy Public API docs. -module docsModule 'application/public-api/publicApiDocs.bicep' = { +module docsModule 'application/public-api/publicApiDocs.bicep' = if (deployDocsSite) { name: 'publicApiDocsModuleDeploy' params: { appSku: docsAppSku @@ -321,7 +325,7 @@ module docsModule 'application/public-api/publicApiDocs.bicep' = { var docsRewriteSetName = '${publicApiResourcePrefix}-docs-rewrites' // Create an Application Gateway to serve public traffic for the Public API Container App. -module appGatewayModule 'application/shared/appGateway.bicep' = if (deployContainerApp) { +module appGatewayModule 'application/shared/appGateway.bicep' = if (deployContainerApp && deployDocsSite) { name: 'appGatewayModuleDeploy' params: { location: location diff --git a/infrastructure/templates/public-api/parameters/main-dev.bicepparam b/infrastructure/templates/public-api/parameters/main-dev.bicepparam index d9b2425967e..38b87677050 100644 --- a/infrastructure/templates/public-api/parameters/main-dev.bicepparam +++ b/infrastructure/templates/public-api/parameters/main-dev.bicepparam @@ -18,8 +18,16 @@ param publicApiContainerAppConfig = { cpuCores: 4 memoryGis: 8 minReplicas: 1 - maxReplicas: 16 + maxReplicas: 100 + scaleAtConcurrentHttpRequests: 5 workloadProfileName: 'Consumption' } +param publicApiContainerAppWorkloadProfiles = [{ + name: 'D8' + workloadProfileType: 'D8' + minimumCount: 0 + maximumCount: 10 +}] + param enableThemeDeletion = true diff --git a/infrastructure/templates/public-api/types.bicep b/infrastructure/templates/public-api/types.bicep index 44ea8544cfa..76ab2098458 100644 --- a/infrastructure/templates/public-api/types.bicep +++ b/infrastructure/templates/public-api/types.bicep @@ -226,10 +226,11 @@ type StaticWebAppSku = 'Free' | 'Standard' @export() type ContainerAppResourceConfig = { workloadProfileName: string - minReplicas: int - maxReplicas: int cpuCores: int memoryGis: int + minReplicas: int + maxReplicas: int + scaleAtConcurrentHttpRequests: int? } @export()