Merge pull request #5432 from dfe-analytical-services/EES-5685-add-av…

…ailability-and-health-alerts-for-public-api

Ees 5685 add availability and health alerts for public api

infrastructure/templates/public-api/application/public-api/publicApiApp.bicep

@@ -30,6 +30,9 @@ param apiAppRegistrationClientId string
 @description('Specifies the Application Insights connection string for this Container App to use for its monitoring.')
 param appInsightsConnectionString string
 
+@description('Whether to create or update Azure Monitor alerts during this deploy')
+param deployAlerts bool
+
 @description('Specifies a set of tags with which to tag the resource in Azure.')
 param tagValues object
 
@@ -146,6 +149,15 @@ module apiContainerAppModule '../../components/containerApp.bicep' = {
   }
 }
 
+module containerAppRestartsAlert '../../components/alerts/containerApps/restarts.bicep' = if (deployAlerts) {
+  name: '${resourceNames.publicApi.apiApp}RestartsDeploy'
+  params: {
+    resourceNames: [resourceNames.publicApi.apiApp]
+    alertsGroupName: resourceNames.existingResources.alertsGroup
+    tagValues: tagValues
+  }
+}
+
 output containerAppFqdn string = apiContainerAppModule.outputs.containerAppFqdn
 output containerAppName string = apiContainerAppModule.outputs.containerAppName
 output healthProbePath string = '/health'

infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep

@@ -6,21 +6,21 @@ param resourceNames ResourceNames
 @description('Specifies the location for all resources.')
 param location string
 
-@description('Alert metric name prefix')
-param metricsNamePrefix string
-
 @description('The Application Insights key that is associated with this resource')
 param applicationInsightsKey string
 
 @description('Specifies whether or not the Data Processor Function App already exists.')
-param dataProcessorFunctionAppExists bool = false
+param dataProcessorFunctionAppExists bool
 
 @description('Specifies the Application (Client) Id of a pre-existing App Registration used to represent the Data Processor Function App.')
 param dataProcessorAppRegistrationClientId string
 
 @description('Public API Storage : Firewall rules.')
 param storageFirewallRules FirewallRule[] = []
 
+@description('Whether to create or update Azure Monitor alerts during this deploy')
+param deployAlerts bool
+
 @description('Specifies a set of tags with which to tag the resource in Azure.')
 param tagValues object
 
@@ -68,7 +68,6 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = {
     functionAppName: resourceNames.publicApi.dataProcessor
     appServicePlanName: resourceNames.publicApi.dataProcessor
     storageAccountsNamePrefix: resourceNames.publicApi.dataProcessorStorageAccountsPrefix
-    alertsGroupName: resourceNames.existingResources.alertsGroup
     location: location
     applicationInsightsKey: applicationInsightsKey
     subnetId: outboundVnetSubnet.id
@@ -98,10 +97,7 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = {
       family: 'EP'
     }
     preWarmedInstanceCount: 1
-    healthCheck: {
-      path: '/api/HealthCheck'
-      unhealthyMetricName: '${metricsNamePrefix}Unhealthy'
-    }
+    healthCheckPath: '/api/HealthCheck'
     appSettings: {
       App__MetaInsertBatchSize: 1000
     }
@@ -117,6 +113,41 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = {
   }
 }
 
+module functionAppHealthAlert '../../components/alerts/sites/healthAlert.bicep' = if (deployAlerts) {
+  name: '${resourceNames.publicApi.dataProcessor}HealthDeploy'
+  params: {
+    resourceNames: [resourceNames.publicApi.dataProcessor]
+    alertsGroupName: resourceNames.existingResources.alertsGroup
+    tagValues: tagValues
+  }
+}
+
+module storageAccountAvailabilityAlerts '../../components/alerts/storageAccounts/availabilityAlert.bicep' = if (deployAlerts) {
+  name: '${resourceNames.publicApi.dataProcessor}StorageAvailabilityDeploy'
+  params: {
+    resourceNames: [
+      dataProcessorFunctionAppModule.outputs.managementStorageAccountName
+      dataProcessorFunctionAppModule.outputs.slot1StorageAccountName
+      dataProcessorFunctionAppModule.outputs.slot2StorageAccountName
+    ]
+    alertsGroupName: resourceNames.existingResources.alertsGroup
+    tagValues: tagValues
+  }
+}
+
+module fileServiceAvailabilityAlerts '../../components/alerts/fileServices/availabilityAlert.bicep' = if (deployAlerts) {
+  name: '${resourceNames.publicApi.dataProcessor}FsAvailabilityDeploy'
+  params: {
+    resourceNames: [
+      dataProcessorFunctionAppModule.outputs.managementStorageAccountName
+      dataProcessorFunctionAppModule.outputs.slot1StorageAccountName
+      dataProcessorFunctionAppModule.outputs.slot2StorageAccountName
+    ]
+    alertsGroupName: resourceNames.existingResources.alertsGroup
+    tagValues: tagValues
+  }
+}
+
 output managedIdentityName string = dataProcessorFunctionAppManagedIdentity.name
 output managedIdentityClientId string = dataProcessorFunctionAppManagedIdentity.properties.clientId
 output publicApiDataFileShareMountPath string = publicApiDataFileShareMountPath

infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep

@@ -6,14 +6,17 @@ param resourceNames ResourceNames
 param location string
 
 @description('Public API Storage : Size of the file share in GB.')
-param publicApiDataFileShareQuota int = 1
+param publicApiDataFileShareQuota int
 
 @description('Public API Storage : Firewall rules.')
-param storageFirewallRules FirewallRule[] = []
+param storageFirewallRules FirewallRule[]
 
 @description('Specifies a set of tags with which to tag the resource in Azure.')
 param tagValues object
 
+@description('Whether to create or update Azure Monitor alerts during this deploy')
+param deployAlerts bool
+
 resource vNet 'Microsoft.Network/virtualNetworks@2023-11-01' existing = {
   name: resourceNames.existingResources.vNet
 }
@@ -56,6 +59,24 @@ module dataFilesFileShareModule '../../components/fileShare.bicep' = {
   }
 }
 
+module storageAccountAvailabilityAlert '../../components/alerts/storageAccounts/availabilityAlert.bicep' = if (deployAlerts) {
+  name: '${resourceNames.publicApi.publicApiStorageAccount}AvailabilityDeploy'
+  params: {
+    resourceNames: [resourceNames.publicApi.publicApiStorageAccount]
+    alertsGroupName: resourceNames.existingResources.alertsGroup
+    tagValues: tagValues
+  }
+}
+
+module fileServiceAvailabilityAlert '../../components/alerts/fileServices/availabilityAlert.bicep' = if (deployAlerts) {
+  name: '${resourceNames.publicApi.publicApiStorageAccount}FsAvailabilityDeploy'
+  params: {
+    resourceNames: [resourceNames.publicApi.publicApiStorageAccount]
+    alertsGroupName: resourceNames.existingResources.alertsGroup
+    tagValues: tagValues
+  }
+}
+
 output storageAccountName string = publicApiStorageAccountModule.outputs.storageAccountName
 output connectionStringSecretName string = publicApiStorageAccountModule.outputs.connectionStringSecretName
 output accessKeySecretName string = publicApiStorageAccountModule.outputs.accessKeySecretName

infrastructure/templates/public-api/application/shared/appGateway.bicep

@@ -24,6 +24,9 @@ param routes AppGatewayRoute[]
 @description('Rules for how the App Gateway should rewrite URLs')
 param rewrites AppGatewayRewriteSet[]
 
+@description('Whether to create or update Azure Monitor alerts during this deploy')
+param deployAlerts bool
+
 @description('Tags for the resources')
 param tagValues object
 
@@ -51,3 +54,12 @@ module appGatewayModule '../../components/appGateway.bicep' = {
     tagValues: tagValues
   }
 }
+
+module backendPoolsHealthAlert '../../components/alerts/appGateways/backendPoolHealth.bicep' = if (deployAlerts) {
+  name: '${resourceNames.sharedResources.appGateway}BackendPoolsHealthDeploy'
+  params: {
+    resourceNames: [resourceNames.sharedResources.appGateway]
+    alertsGroupName: resourceNames.existingResources.alertsGroup
+    tagValues: tagValues
+  }
+}

infrastructure/templates/public-api/application/shared/postgreSqlFlexibleServer.bicep

@@ -31,6 +31,9 @@ param privateEndpointSubnetId string
 @description('An array of Entra ID admin principal names for this resource')
 param entraIdAdminPrincipals PrincipalNameAndId[] = []
 
+@description('Whether to create or update Azure Monitor alerts during this deploy')
+param deployAlerts bool
+
 @description('Specifies a set of tags with which to tag the resource in Azure.')
 param tagValues object
 
@@ -70,6 +73,15 @@ resource maxPreparedTransactionsConfig 'Microsoft.DBforPostgreSQL/flexibleServer
   ]
 }
 
+module databaseAliveAlert '../../components/alerts/flexibleServers/databaseAlive.bicep' = if (deployAlerts) {
+  name: '${resourceNames.sharedResources.postgreSqlFlexibleServer}DbAliveDeploy'
+  params: {
+    resourceNames: [resourceNames.sharedResources.postgreSqlFlexibleServer]
+    alertsGroupName: resourceNames.existingResources.alertsGroup
+    tagValues: tagValues
+  }
+}
+
 var managedIdentityConnectionStringTemplate = postgreSqlServerModule.outputs.managedIdentityConnectionStringTemplate
 
 var dataProcessorPsqlConnectionStringSecretKey = 'ees-publicapi-data-processor-connectionstring-publicdatadb'

infrastructure/templates/public-api/ci/azure-pipelines.yml

@@ -7,6 +7,9 @@ parameters:
   - name: updatePsqlFlexibleServer
     displayName: Does the PostgreSQL Flexible Server require any updates? False by default to avoid unnecessarily lengthy deploys.
     default: false
+  - name: deployAlerts
+    displayName: Whether to create or update Azure Monitor alerts during this deploy.
+    default: false
   - name: forceDeployToEnvironment
     displayName: Set to either dev or test to force a deploy to that environment from the chosen branch.
     type: string
@@ -42,6 +45,8 @@ variables:
     value: ${{ parameters.deployContainerApp }}
   - name: updatePsqlFlexibleServer
     value: ${{ parameters.updatePsqlFlexibleServer }}
+  - name: deployAlerts
+    value: ${{ parameters.deployAlerts }}
 
 pool:
   vmImage: $(vmImageName)

infrastructure/templates/public-api/ci/jobs/deploy-infrastructure.yml

@@ -37,6 +37,7 @@ jobs:
                 parameterFile: $(paramFile)
                 deployContainerApp: true
                 updatePsqlFlexibleServer: false
+                deployAlerts: false
                 dataProcessorExists: true
 
             - task: AzureCLI@2
@@ -63,6 +64,7 @@ jobs:
                 parameterFile: $(paramFile)
                 deployContainerApp: $(deployContainerApp)
                 updatePsqlFlexibleServer: $(updatePsqlFlexibleServer)
+                deployAlerts: $(deployAlerts)
                 dataProcessorExists: $(dataProcessorExists)
 
            # - template: ../tasks/assign-app-role-to-service-principal.yml

infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml

@@ -13,13 +13,12 @@ parameters:
     type: string
   - name: deployContainerApp
     type: string
-    default: true
   - name: updatePsqlFlexibleServer
     type: string
-    default: false
+  - name: deployAlerts
+    type: string
   - name: dataProcessorExists
     type: string
-    default: true
 
 steps:
   - task: AzureCLI@2
@@ -48,6 +47,7 @@ steps:
               dockerImagesTag='$(resources.pipeline.MainBuild.runName)' \
               deployContainerApp=${{ parameters.deployContainerApp }} \
               updatePsqlFlexibleServer=${{ parameters.updatePsqlFlexibleServer }} \
+              deployAlerts=${{ parameters.deployAlerts }} \
               dataProcessorFunctionAppExists=${{ parameters.dataProcessorExists }} \
               dataProcessorAppRegistrationClientId='$(dataProcessorAppRegistrationClientId)' \
               apiAppRegistrationClientId='$(apiAppRegistrationClientId)'

infrastructure/templates/public-api/components/alerts/appGateways/backendPoolHealth.bicep

@@ -0,0 +1,33 @@
+import { Severity } from '../types.bicep'
+
+@description('Names of the resources that these alerts are being applied to.')
+param resourceNames string[]
+
+@description('The alert severity.')
+param severity Severity = 'Critical'
+
+@description('Name of the Alerts Group used to send alert messages.')
+param alertsGroupName string
+
+@description('Tags with which to tag the resource in Azure.')
+param tagValues object
+
+module alerts '../staticMetricAlert.bicep' = [for name in resourceNames: {
+  name: '${name}BackendHealthAlertModule'
+  params: {
+    alertName: '${name}-backend-pool-health'
+    resourceIds: [resourceId('Microsoft.Network/applicationGateways', name)]
+    resourceType: 'Microsoft.Network/applicationGateways'
+    query: {
+      metric: 'UnhealthyHostCount'
+      aggregation: 'Total'
+      operator: 'GreaterThan'
+      threshold: 0
+    }
+    evaluationFrequency: 'PT1M'
+    windowSize: 'PT5M'
+    severity: severity
+    alertsGroupName: alertsGroupName
+    tagValues: tagValues
+  }
+}]

infrastructure/templates/public-api/components/alerts/containerApps/restarts.bicep

@@ -0,0 +1,33 @@
+import { Severity } from '../types.bicep'
+
+@description('Names of the resources that these alerts are being applied to.')
+param resourceNames string[]
+
+@description('The alert severity.')
+param severity Severity = 'Warning'
+
+@description('Name of the Alerts Group used to send alert messages.')
+param alertsGroupName string
+
+@description('Tags with which to tag the resource in Azure.')
+param tagValues object
+
+module alerts '../staticMetricAlert.bicep' = [for name in resourceNames: {
+  name: '${name}RestartsAlertModule'
+  params: {
+    alertName: '${name}-restarts'
+    resourceIds: [resourceId('Microsoft.App/containerApps', name)]
+    resourceType: 'Microsoft.App/containerApps'
+    query: {
+      metric: 'RestartCount'
+      aggregation: 'Total'
+      operator: 'GreaterThan'
+      threshold: 0
+    }
+    evaluationFrequency: 'PT1M'
+    windowSize: 'PT5M'
+    severity: severity
+    alertsGroupName: alertsGroupName
+    tagValues: tagValues
+  }
+}]

infrastructure/templates/public-api/components/alerts/fileServices/availabilityAlert.bicep

@@ -0,0 +1,33 @@
+import { Severity } from '../types.bicep'
+
+@description('Names of the resources that these alerts are being applied to.')
+param resourceNames string[]
+
+@description('The alert severity.')
+param severity Severity = 'Critical'
+
+@description('Name of the Alerts Group used to send alert messages.')
+param alertsGroupName string
+
+@description('Tags with which to tag the resource in Azure.')
+param tagValues object
+
+module alerts '../staticMetricAlert.bicep' = [for name in resourceNames: {
+  name: '${name}FsAvailabilityAlertModule'
+  params: {
+    alertName: '${name}-fileservice-availability'
+    resourceIds: [resourceId('Microsoft.Storage/storageAccounts/fileServices', name, 'default')]
+    resourceType: 'Microsoft.Storage/storageAccounts/fileServices'
+    query: {
+      metric: 'availability'
+      aggregation: 'Average'
+      operator: 'LessThan'
+      threshold: 100
+    }
+    evaluationFrequency: 'PT1M'
+    windowSize: 'PT5M'
+    severity: severity
+    alertsGroupName: alertsGroupName
+    tagValues: tagValues
+  }
+}]