diff --git a/azuredeploy.bicep b/azuredeploy.bicep index 05a833c4..da92b4d0 100644 --- a/azuredeploy.bicep +++ b/azuredeploy.bicep @@ -22,13 +22,6 @@ param ingestionIdName string @description('Name of the package managed identity') param packageIdName string -@description('Client ID (used by cloudprovider)') -param servicePrincipalClientId string - -@description('The Service Principal Client Secret.') -@secure() -param servicePrincipalClientSecret string - @description('The type of operating system.') @allowed([ 'Linux' @@ -40,30 +33,19 @@ param osType string = 'Linux' @maxValue(1023) param osDiskSizeGB int = 0 -@description('The version of Kubernetes. It must be supported in the target location.') -param kubernetesVersion string +//@description('The version of Kubernetes. It must be supported in the target location.') +//param kubernetesVersion string -@description('Type of the storage account that will store Redis Cache.') -@allowed([ - 'Standard_LRS' - 'Standard_ZRS' - 'Standard_GRS' -]) -param deliveryRedisStorageType string = 'Standard_LRS' +param logAnalyticsWorkspaceID string -var clusterNamePrefix = 'aks' var managedIdentityOperatorRoleId = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'f1a07417-d97a-45cb-824c-7a7467783830') -var deliveryRedisStorageName = 'rsto${uniqueString(resourceGroup().id)}' var nestedACRDeploymentName = 'azuredeploy-acr-${acrResourceGroupName}' -var aksLogAnalyticsNamePrefix = 'logsAnalytics' var monitoringMetricsPublisherRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '3913510d-42f4-4e42-8a64-420c390055eb') +var contributorRoleId = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c') var nodeResourceGroupName = 'rg-${aksClusterName}-nodepools' -var aksClusterName = uniqueString(clusterNamePrefix, resourceGroup().id) +var aksClusterName = 'aks-${uniqueString(resourceGroup().id)}' var agentCount = 2 var agentVMSize = 'Standard_D2_v2' -var workspaceName = 'la-${uniqueString(aksLogAnalyticsNamePrefix, resourceGroup().id)}' -var workspaceSku = 'pergb2018' -var workspaceRetentionInDays = 0 module nestedACRDeployment './azuredeploy_nested_nestedACRDeployment.bicep' = { name: nestedACRDeploymentName @@ -74,26 +56,23 @@ module nestedACRDeployment './azuredeploy_nested_nestedACRDeployment.bicep' = { } } -resource workspace 'Microsoft.OperationalInsights/workspaces@2022-10-01' = { - name: workspaceName - location: location - properties: { - retentionInDays: workspaceRetentionInDays - sku: { - name: workspaceSku - } - features: { - searchVersion: 1 - } - } -} - // The control plane identity used by the cluster. Used for networking access (VNET joining and DNS updating) resource miClusterControlPlane 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { name: 'mi-${aksClusterName}-controlplane' location: location } +//provide contributor role to the RG to AKS managed identity. +resource roleAssignment 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = { + name: guid(resourceGroup().id, miClusterControlPlane.id) + scope: resourceGroup() + properties: { + principalId: miClusterControlPlane.properties.principalId + roleDefinitionId: contributorRoleId + principalType: 'ServicePrincipal' + } +} + resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-09-02-preview' = { name: aksClusterName location: location @@ -101,7 +80,7 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-09-02-previ environment: 'shared cluster' } properties: { - kubernetesVersion: kubernetesVersion +// kubernetesVersion: kubernetesVersion nodeResourceGroup: nodeResourceGroupName dnsPrefix: aksClusterName agentPoolProfiles: [ @@ -122,14 +101,10 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-09-02-previ mode: 'User' } ] - servicePrincipalProfile: { - clientId: servicePrincipalClientId - secret: servicePrincipalClientSecret - } addonProfiles: { omsagent: { config: { - logAnalyticsWorkspaceResourceID: workspace.id + logAnalyticsWorkspaceResourceID: logAnalyticsWorkspaceID } enabled: true } @@ -174,20 +149,6 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-09-02-previ '${miClusterControlPlane.id}': {} } } - -} - -resource deliveryRedisStorage 'Microsoft.Storage/storageAccounts@2022-09-01' = { - name: deliveryRedisStorageName - sku: { - name: deliveryRedisStorageType - } - kind: 'Storage' - location: location - tags: { - displayName: 'Storage account for inflight deliveries' - app: 'fabrikam-delivery' - } } resource clusterIdentityPublisherRoleAssigment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { diff --git a/charts/package/templates/package-deploy.yaml b/charts/package/templates/package-deploy.yaml index 384cc8f9..ad48d4d0 100644 --- a/charts/package/templates/package-deploy.yaml +++ b/charts/package/templates/package-deploy.yaml @@ -101,6 +101,11 @@ spec: secretKeyRef: name: package-secrets key: appinsights-ikey + - name: APPINSIGHTS_CONNECTION_STRING + valueFrom: + secretKeyRef: + name: package-secrets + key: appinsights-connstr - name: LOG_LEVEL value: {{ .Values.log.level }} - name: CONTAINER_NAME @@ -114,4 +119,4 @@ spec: driver: secrets-store.csi.k8s.io readOnly: true volumeAttributes: - secretProviderClass: package-secrets-csi-akv-{{ $svcversion }} \ No newline at end of file + secretProviderClass: package-secrets-csi-akv-{{ $svcversion }} diff --git a/charts/package/templates/package-secret-provider.yaml b/charts/package/templates/package-secret-provider.yaml index 74302c0a..90d37c39 100644 --- a/charts/package/templates/package-secret-provider.yaml +++ b/charts/package/templates/package-secret-provider.yaml @@ -22,6 +22,8 @@ spec: key: cosmosdb-connstr - objectName: ApplicationInsights--InstrumentationKey key: appinsights-ikey + - objectName: ApplicationInsights--ConnectionString + key: appinsights-connstr parameters: usePodIdentity: "false" clientID: {{ .Values.identity.clientid }} @@ -35,6 +37,10 @@ spec: - | objectName: ApplicationInsights--InstrumentationKey objectAlias: ApplicationInsights--InstrumentationKey - objectType: secret + objectType: secret + - | + objectName: ApplicationInsights--ConnectionString + objectAlias: ApplicationInsights--ConnectionString + objectType: secret tenantId: {{ .Values.identity.tenantId }} --- diff --git a/deployment.md b/deployment.md index 10105917..e9c3acf7 100644 --- a/deployment.md +++ b/deployment.md @@ -5,14 +5,13 @@ - Azure subscription > Important: The user initiating the deployment process must have access to the **Microsoft.Authorization/roleAssignments/write** permission. For more information, see [the Container Insights doc](https://docs.microsoft.com/azure/azure-monitor/insights/container-insights-troubleshoot#authorization-error-during-onboarding-or-update-operation) - [Azure CLI 2.53.1 or newer](https://docs.microsoft.com/cli/azure/install-azure-cli) -- [Docker](https://docs.docker.com/) - [JQ](https://stedolan.github.io/jq/download/) +- Kubectl +(az aks install-cli) +- Helm +(curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash) -> Note: in linux systems, it is possible to run the docker command without prefacing -> with sudo. For more information, please refer to [the Post-installation steps -> for linux](https://docs.docker.com/install/linux/linux-postinstall/) - -Clone or download this repo locally. +## Clone or download this repo locally. ```bash git clone --recurse-submodules https://github.com/mspnp/microservices-reference-implementation.git && \ @@ -21,99 +20,109 @@ cd microservices-reference-implementation/ The deployment steps shown here use Bash shell commands. On Windows, you can use the [Windows Subsystem for Linux](https://docs.microsoft.com/windows/wsl/about) to run Bash. -## Azure Resources Provisioning - Set environment variables. ```bash export LOCATION=eastus2 ``` -Log in in to Azure. +### Log in to Azure CLI ```bash az login +``` -# if you have several subscriptions, select one -# az account set -s +### Deploy the workload's prerequisites + +```bash +az deployment sub create --name workload-stamp-prereqs --location ${LOCATION} --template-file ./workload/workload-stamp-prereqs.bicep --parameters resourceGroupLocation=${LOCATION} ``` -## Deployment +:book: This pre-flight Bicep template creates two resource groups. Additionally five User Identites are provisioned that will be later associated to every containerized microservice. This is because they will need Azure RBAC roles over the Azure KeyVault to read secrets in runtime. The resources will be created on the resouce group location and each resource group will contain the region as part of their names -> Note: this deployment might take up to 20 minutes +### Get the workload user assigned identities -Infrastructure +```bash +DELIVERY_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-delivery --query principalId -o tsv) && \ +DRONESCHEDULER_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-dronescheduler --query principalId -o tsv) && \ +WORKFLOW_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-workflow --query principalId -o tsv) && \ +PACKAGE_ID_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-package --query principalId -o tsv) && \ +INGESTION_ID_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-ingestion --query principalId -o tsv) +``` + +### Deploy the workload related resources ```bash -# Deploy the managed identities (This takes less than two minutes.) +az deployment group create -f ./workload/workload-stamp.bicep -g rg-shipping-dronedelivery-${LOCATION} -p droneSchedulerPrincipalId=$DRONESCHEDULER_PRINCIPAL_ID -p workflowPrincipalId=$WORKFLOW_PRINCIPAL_ID -p deliveryPrincipalId=$DELIVERY_PRINCIPAL_ID -p ingestionPrincipalId=$INGESTION_ID_PRINCIPAL_ID -p packagePrincipalId=$PACKAGE_ID_PRINCIPAL_ID +``` -export PREREQS_DEPLOYMENT_NAME=workload-stamp-prereqs-main +### Assign ACR variables -az deployment sub create --name $PREREQS_DEPLOYMENT_NAME --location ${LOCATION} --template-file ./workload/workload-stamp-prereqs.bicep --parameters resourceGroupLocation=${LOCATION} +```bash +ACR_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.acrName.value -o tsv) +ACR_SERVER=$(az acr show -n $ACR_NAME --query loginServer -o tsv) +``` -# Get the user identities -export DELIVERY_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-delivery --query principalId -o tsv) && \ -export DRONESCHEDULER_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-dronescheduler --query principalId -o tsv) && \ -export WORKFLOW_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-workflow --query principalId -o tsv) && \ -export PACKAGE_ID_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-package --query principalId -o tsv) && \ -export INGESTION_ID_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-ingestion --query principalId -o tsv) +## Build the microservice images +### Steps -# Wait for Microsoft Entra ID propagation -until az ad sp show --id $DELIVERY_PRINCIPAL_ID &> /dev/null ; do echo "Waiting for Microsoft Entra ID propagation" && sleep 5; done -until az ad sp show --id $DRONESCHEDULER_PRINCIPAL_ID &> /dev/null ; do echo "Waiting for Microsoft Entra ID propagation" && sleep 5; done -until az ad sp show --id $WORKFLOW_PRINCIPAL_ID &> /dev/null ; do echo "Waiting for Microsoft Entra ID propagation" && sleep 5; done -until az ad sp show --id $PACKAGE_ID_PRINCIPAL_ID &> /dev/null ; do echo "Waiting for Microsoft Entra ID propagation" && sleep 5; done -until az ad sp show --id $INGESTION_ID_PRINCIPAL_ID &> /dev/null ; do echo "Waiting for Microsoft Entra ID propagation" && sleep 5; done +1. Build and push the Delivery service container image to ACR. -# Deploy all the workload related resources (This step takes about 10 minutes) -az deployment group create -f ./workload/workload-stamp.bicep -g rg-shipping-dronedelivery-${LOCATION} -p droneSchedulerPrincipalId=$DRONESCHEDULER_PRINCIPAL_ID -p workflowPrincipalId=$WORKFLOW_PRINCIPAL_ID -p deliveryPrincipalId=$DELIVERY_PRINCIPAL_ID -p ingestionPrincipalId=$INGESTION_ID_PRINCIPAL_ID -p packagePrincipalId=$PACKAGE_ID_PRINCIPAL_ID +```bash +az acr build -r $ACR_NAME -t $ACR_SERVER/delivery:0.1.0 ./workload/src/shipping/delivery/. +``` -# Get outputs from workload deploy -export ACR_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.acrName.value -o tsv) -export ACR_SERVER=$(az acr show -n $ACR_NAME --query loginServer -o tsv) +2. Build and push the Ingestion service container image to ACR. + +```bash +az acr build -r $ACR_NAME -t $ACR_SERVER/ingestion:0.1.0 ./workload/src/shipping/ingestion/. ``` -Deploy the managed cluster and all related resources (This step takes about 15 minutes) +3. Build and push the Workflow service container image to ACR. ```bash -export RESOURCE_GROUP_ID=$(az group show --name rg-shipping-dronedelivery-${LOCATION} --query id --output tsv) +az acr build -r $ACR_NAME -t $ACR_SERVER/workflow:0.1.0 ./workload/src/shipping/workflow/. +``` -export SP_DETAILS=$(az ad sp create-for-rbac --role="Contributor" --scopes $RESOURCE_GROUP_ID -o json) && \ -export SP_APP_ID=$(echo $SP_DETAILS | jq ".appId" -r) && \ -export SP_CLIENT_SECRET=$(echo $SP_DETAILS | jq ".password" -r) -export TENANT_ID=$(az account show --query tenantId --output tsv) +4. Build and push the DroneScheduler service container image to ACR. -export DEPLOYMENT_SUFFIX=$(date +%S%N) +```bash +az acr build -r $ACR_NAME -f ./workload/src/shipping/dronescheduler/Dockerfile -t $ACR_SERVER/dronescheduler:0.1.0 ./workload/src/shipping/. +``` -export KUBERNETES_VERSION=$(az aks get-versions -l $LOCATION --query "values[?isDefault].version" -o tsv) +5. Build and push the Package service container image to ACR. -export DEPLOYMENT_NAME=azuredeploy-$DEPLOYMENT_SUFFIX -az deployment group create -g rg-shipping-dronedelivery-${LOCATION} --name $DEPLOYMENT_NAME --template-file azuredeploy.bicep \ ---parameters servicePrincipalClientId=$SP_APP_ID \ - servicePrincipalClientSecret=$SP_CLIENT_SECRET \ - kubernetesVersion=$KUBERNETES_VERSION \ - deliveryIdName=uid-delivery \ +```bash +az acr build -r $ACR_NAME -t $ACR_SERVER/package:0.1.0 ./workload/src/shipping/package/. +``` + +## Deploy the managed cluster and related resources + +```bash + +export LOG_ANALYTICS_WORKSPACE_ID=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.laWorkspace.value -o tsv) + +az deployment group create -g rg-shipping-dronedelivery-${LOCATION} --name managed-cluster-deployment --template-file azuredeploy.bicep \ +--parameters deliveryIdName=uid-delivery \ ingestionIdName=uid-ingestion \ packageIdName=uid-package \ droneSchedulerIdName=uid-dronescheduler \ workflowIdName=uid-workflow \ acrResourceGroupName=rg-shipping-dronedelivery-${LOCATION}-acr \ - acrName=$ACR_NAME + acrName=$ACR_NAME \ + logAnalyticsWorkspaceID=$LOG_ANALYTICS_WORKSPACE_ID ``` -Get the cluster name output from Azure Deploy. +### Get the cluster name output from Azure Deploy. ```bash -export CLUSTER_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n $DEPLOYMENT_NAME --query properties.outputs.aksClusterName.value -o tsv) -echo $CLUSTER_NAME +export CLUSTER_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n managed-cluster-deployment --query properties.outputs.aksClusterName.value -o tsv) ``` -Download kubectl and create a Kubernetes namespace. +### Get the AKS cluster credentials and create a Kubernetes namespace. ```bash -# Install kubectl -az aks install-cli # Get the Kubernetes cluster credentials az aks get-credentials --resource-group=rg-shipping-dronedelivery-${LOCATION} --name=$CLUSTER_NAME @@ -122,55 +131,60 @@ az aks get-credentials --resource-group=rg-shipping-dronedelivery-${LOCATION} -- kubectl create namespace backend-dev ``` -Install and initialize Helm. +### Configure RBAC permissions for Azure application insights. ```bash -# install helm 3 -curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash -``` - -Integrate Application Insights instance. - -```bash -# Acquire Instrumentation Key -export AI_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.appInsightsName.value -o tsv) -echo $AI_NAME # add RBAC for AppInsights kubectl apply -f k8s/k8s-rbac-ai.yaml ``` -## Verify that the secrets-store pods are running in the kube-system namespace +### Set resource quotas for the namespace. ```bash -kubectl get pods -n kube-system +kubectl apply -f k8s/k8s-resource-quotas-dev.yaml ``` -You should see an output similar to this: +### Get the OIDC Issuer URL, Tenant ID, and Object ID of the signed-in user. ```bash -NAME READY STATUS RESTARTS AGE -aks-secrets-store-csi-driver-4bjzx 3/3 Running 2 28m -aks-secrets-store-csi-driver-b22bj 3/3 Running 1 28m -aks-secrets-store-provider-azure-2k5mx 1/1 Running 0 28m -aks-secrets-store-provider-azure-l5w98 1/1 Running 0 28m +export AKS_OIDC_ISSUER="$(az aks show -n $CLUSTER_NAME -g rg-shipping-dronedelivery-${LOCATION} --query "oidcIssuerProfile.issuerUrl" -otsv)" +export TENANT_ID=$(az account show --query tenantId --output tsv) +export SIGNED_IN_OBJECT_ID=$(az ad signed-in-user show --query 'id' -o tsv) +``` + +### Create manage identity federations for microservices. ``` +# Setup managed identity for delivery microservice to trust your Kubernetes service account +az identity federated-credential create --name credential-for-delivery --identity-name uid-delivery --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:delivery-sa-v0.1.0 + +# Setup managed identity for package microservice to trust your Kubernetes service account +az identity federated-credential create --name credential-for-package --identity-name uid-package --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:package-sa-v0.1.0 + +# Setup your managed identity to trust your Kubernetes service account +az identity federated-credential create --name credential-for-workflow --identity-name uid-workflow --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:workflow-sa-v0.1.0 + +# Setup your managed identity to trust your Kubernetes service account +az identity federated-credential create --name credential-for-ingestion --identity-name uid-ingestion --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:ingestion-sa-v0.1.0 -## Collect details of managed ingress controller. +# Setup your managed identity to trust your Kubernetes service account +az identity federated-credential create --name credential-for-dronescheduler --identity-name uid-dronescheduler --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:dronescheduler-sa-v0.1.0 +``` + +### Collect details of managed ingress controller. ```bash # Obtain the load balancer ip address of managed ingress and assign a domain name export INGRESS_LOAD_BALANCER_IP=$(kubectl get service -n app-routing-system nginx -o jsonpath="{.status.loadBalancer.ingress[0].ip}" 2> /dev/null) - export INGRESS_LOAD_BALANCER_IP_ID=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$INGRESS_LOAD_BALANCER_IP')].[id]" --output tsv) && \ export EXTERNAL_INGEST_DNS_NAME="dronedelivery-${LOCATION}-${RANDOM}-ing" && \ export EXTERNAL_INGEST_FQDN=$(az network public-ip update --ids $INGRESS_LOAD_BALANCER_IP_ID --dns-name $EXTERNAL_INGEST_DNS_NAME --query "dnsSettings.fqdn" --output tsv) ``` -## Create self-signed certificate for TLS +### Create self-signed certificate for TLS > :warning: WARNING > @@ -179,7 +193,6 @@ export EXTERNAL_INGEST_FQDN=$(az network public-ip update --ids $INGRESS_LOAD_BA > For your production cluster, use your > security best practices for digital certificates creation and lifetime management. - ```bash # Create a self-signed certificate for TLS @@ -189,18 +202,6 @@ openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ -subj "/CN=${EXTERNAL_INGEST_FQDN}/O=fabrikam" ``` -## Setup cluster resource quota - -```bash -kubectl apply -f k8s/k8s-resource-quotas-dev.yaml -``` - -## Get the OIDC Issuer URL - -```bash -export AKS_OIDC_ISSUER="$(az aks show -n $CLUSTER_NAME -g rg-shipping-dronedelivery-${LOCATION} --query "oidcIssuerProfile.issuerUrl" -otsv)" -``` - ## Deploy the Delivery service Extract resource details from deployment. @@ -212,33 +213,19 @@ export COLLECTION_NAME="${DATABASE_NAME}-col" && \ export DELIVERY_KEYVAULT_URI=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.deliveryKeyVaultUri.value -o tsv) && \ export DELIVERY_KEYVAULT_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.deliveryKeyVaultName.value -o tsv) && \ export DELIVERY_PRINCIPAL_CLIENT_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-delivery --query clientId -o tsv) -``` - -Build and publish the Delivery service container image. - -```bash -az acr build -r $ACR_NAME -t $ACR_SERVER/delivery:0.1.0 ./workload/src/shipping/delivery/. -``` - -Deploy the Delivery service. +export DELIVERY_KEYVAULT_ID=$(az resource show -g rg-shipping-dronedelivery-${LOCATION} -n $DELIVERY_KEYVAULT_NAME --resource-type 'Microsoft.KeyVault/vaults' --query id --output tsv) -```bash # Create secrets -# Note: Ingress TLS key and certificate secrets cannot be exported as outputs in ARM deployments # The current user is given permission to import secrets and then it is deleted right after the secret creation command is executed -export SIGNED_IN_OBJECT_ID=$(az ad signed-in-user show --query 'id' -o tsv) -export DELIVERY_KEYVAULT_ID=$(az resource show -g rg-shipping-dronedelivery-${LOCATION} -n $DELIVERY_KEYVAULT_NAME --resource-type 'Microsoft.KeyVault/vaults' --query id --output tsv) az role assignment create --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $DELIVERY_KEYVAULT_ID +#wait for role assignment to finish. +sleep 30 az keyvault secret set --name Delivery-Ingress-Tls-Key --vault-name $DELIVERY_KEYVAULT_NAME --value "$(cat ingestion-ingress-tls.key)" az keyvault secret set --name Delivery-Ingress-Tls-Crt --vault-name $DELIVERY_KEYVAULT_NAME --value "$(cat ingestion-ingress-tls.crt)" - az role assignment delete --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $DELIVERY_KEYVAULT_ID -#Setup your managed identity to trust your Kubernetes service account -az identity federated-credential create --name credential-for-delivery --identity-name uid-delivery --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:delivery-sa-v0.1.0 - # Deploy the service helm package charts/delivery/ -u && \ helm install delivery-v0.1.0-dev delivery-v0.1.0.tgz \ @@ -254,7 +241,7 @@ helm install delivery-v0.1.0-dev delivery-v0.1.0.tgz \ --set identity.tenantId=$TENANT_ID \ --set keyVaultName=$DELIVERY_KEYVAULT_NAME \ --set ingress.tls=true \ - --set ingress.class=nginx \ + --set ingress.class=webapprouting.kubernetes.azure.com \ --set cosmosdb.id=$DATABASE_NAME \ --set cosmosdb.collectionid=$COLLECTION_NAME \ --set keyvault.uri=$DELIVERY_KEYVAULT_URI \ @@ -263,7 +250,7 @@ helm install delivery-v0.1.0-dev delivery-v0.1.0.tgz \ --namespace backend-dev \ --dependency-update -# Verify the pod is created +# Verify the helm deployment status. helm status delivery-v0.1.0-dev --namespace backend-dev ``` @@ -272,36 +259,10 @@ helm status delivery-v0.1.0-dev --namespace backend-dev Extract resource details from deployment. ```bash -export COSMOSDB_NAME_PACKAGE=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.packageMongoDbName.value -o tsv) export PACKAGE_KEYVAULT_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.packageKeyVaultName.value -o tsv) export PACKAGE_ID_CLIENT_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-package --query clientId -o tsv) -``` - -Build the Package service. - -```bash -az acr build -r $ACR_NAME -t $ACR_SERVER/package:0.1.0 ./workload/src/shipping/package/. -``` - -Deploy the Package service. - -```bash -# Create secret -# Note: Connection strings cannot be exported as outputs in ARM deployments -# The current user is given permission to import secrets and then it is deleted right after the secret creation command is executed -export COSMOSDB_CONNECTION_PACKAGE=$(az cosmosdb keys list --type connection-strings --name $COSMOSDB_NAME_PACKAGE --resource-group rg-shipping-dronedelivery-${LOCATION} --query "connectionStrings[0].connectionString" -o tsv | sed 's/==/%3D%3D/g') && \ export COSMOSDB_COL_NAME_PACKAGE=packages -export PACKAGE_KEYVAULT_ID=$(az resource show -g rg-shipping-dronedelivery-${LOCATION} -n $PACKAGE_KEYVAULT_NAME --resource-type 'Microsoft.KeyVault/vaults' --query id --output tsv) -az role assignment create --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $PACKAGE_KEYVAULT_ID - -az keyvault secret set --name CosmosDb--ConnectionString --vault-name $PACKAGE_KEYVAULT_NAME --value $COSMOSDB_CONNECTION_PACKAGE - -az role assignment delete --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $PACKAGE_KEYVAULT_ID - -# Setup your managed identity to trust your Kubernetes service account -az identity federated-credential create --name credential-for-package --identity-name uid-package --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:package-sa-v0.1.0 - # Deploy service helm package charts/package/ -u && \ helm install package-v0.1.0-dev package-v0.1.0.tgz \ @@ -314,7 +275,7 @@ helm install package-v0.1.0-dev package-v0.1.0.tgz \ --set ingress.hosts[0].name=$EXTERNAL_INGEST_FQDN \ --set ingress.hosts[0].serviceName=package \ --set ingress.hosts[0].tls=false \ - --set ingress.class=nginx \ + --set ingress.class=webapprouting.kubernetes.azure.com \ --set cosmosDb.collectionName=$COSMOSDB_COL_NAME_PACKAGE \ --set dockerregistry=$ACR_SERVER \ --set reason="Initial deployment" \ @@ -338,20 +299,6 @@ export WORKFLOW_NAMESPACE_NAME=$(az deployment group show -g rg-shipping-dronede export WORKFLOW_NAMESPACE_ENDPOINT=$(az servicebus namespace show -g rg-shipping-dronedelivery-${LOCATION} -n $WORKFLOW_NAMESPACE_NAME --query serviceBusEndpoint -o tsv) export WORKFLOW_NAMESPACE_SAS_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.workflowServiceAccessKeyName.value -o tsv) -``` - -Build the workflow service. - -```bash -az acr build -r $ACR_NAME -t $ACR_SERVER/workflow:0.1.0 ./workload/src/shipping/workflow/. -``` - -Deploy the Workflow service. - -```bash -# Setup your managed identity to trust your Kubernetes service account -az identity federated-credential create --name credential-for-workflow --identity-name uid-workflow --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:workflow-sa-v0.1.0 - # Deploy the service helm package charts/workflow/ -u && \ helm install workflow-v0.1.0-dev workflow-v0.1.0.tgz \ @@ -387,28 +334,17 @@ export INGESTION_QUEUE_NAMESPACE=$(az deployment group show -g rg-shipping-drone export INGESTION_QUEUE_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.ingestionQueueName.value -o tsv) export INGESTION_KEYVAULT_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.ingestionKeyVaultName.value -o tsv) export INGESTION_ID_CLIENT_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-ingestion --query clientId -o tsv) +export INGESTION_KEYVAULT_ID=$(az resource show -g rg-shipping-dronedelivery-${LOCATION} -n $INGESTION_KEYVAULT_NAME --resource-type 'Microsoft.KeyVault/vaults' --query id --output tsv) # The current user is given permission to import secrets and then it is deleted right after the secret creation command is executed -export INGESTION_KEYVAULT_ID=$(az resource show -g rg-shipping-dronedelivery-${LOCATION} -n $INGESTION_KEYVAULT_NAME --resource-type 'Microsoft.KeyVault/vaults' --query id --output tsv) az role assignment create --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $INGESTION_KEYVAULT_ID +# wait a while for the role propagation to finish. +sleep 30 az keyvault secret set --name Ingestion-Ingress-Tls-Key --vault-name $INGESTION_KEYVAULT_NAME --value "$(cat ingestion-ingress-tls.key)" az keyvault secret set --name Ingestion-Ingress-Tls-Crt --vault-name $INGESTION_KEYVAULT_NAME --value "$(cat ingestion-ingress-tls.crt)" az role assignment delete --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $INGESTION_KEYVAULT_ID -``` - -Build the Ingestion service. - -```bash -az acr build -r $ACR_NAME -t $ACR_SERVER/ingestion:0.1.0 ./workload/src/shipping/ingestion/. -``` - -Deploy the Ingestion service - -```bash -# Setup your managed identity to trust your Kubernetes service account -az identity federated-credential create --name credential-for-ingestion --identity-name uid-ingestion --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:ingestion-sa-v0.1.0 # Deploy service helm package charts/ingestion/ -u && \ @@ -425,7 +361,7 @@ helm install ingestion-v0.1.0-dev ingestion-v0.1.0.tgz \ --set ingress.hosts[0].tls=true \ --set ingress.hosts[0].tlsSecretName=ingestion-ingress-tls \ --set ingress.tls=true \ - --set ingress.class=nginx \ + --set ingress.class=webapprouting.kubernetes.azure.com \ --set secrets.queue.keyname=IngestionServiceAccessKey \ --set secrets.queue.name=${INGESTION_QUEUE_NAME} \ --set secrets.queue.namespace=${INGESTION_QUEUE_NAMESPACE} \ @@ -450,19 +386,6 @@ export AUTH_KEY=$(az cosmosdb keys list -n $DRONESCHEDULER_COSMOSDB_NAME -g rg-s export DRONESCHEDULER_CLIENT_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-dronescheduler --query clientId -o tsv) && \ export DATABASE_NAME="invoicing" && \ export COLLECTION_NAME="utilization" -``` - -Build and publish the container image. - -```bash -az acr build -r $ACR_NAME -f ./workload/src/shipping/dronescheduler/Dockerfile -t $ACR_SERVER/dronescheduler:0.1.0 ./workload/src/shipping/. -``` - -Deploy the dronescheduler service. - -```bash -# Setup your managed identity to trust your Kubernetes service account -az identity federated-credential create --name credential-for-dronescheduler --identity-name uid-dronescheduler --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:dronescheduler-sa-v0.1.0 # Deploy the service helm package charts/dronescheduler/ -u && \ @@ -473,7 +396,7 @@ helm install dronescheduler-v0.1.0-dev dronescheduler-v0.1.0.tgz \ --set ingress.hosts[0].name=$EXTERNAL_INGEST_FQDN \ --set ingress.hosts[0].serviceName=dronescheduler \ --set ingress.hosts[0].tls=false \ - --set ingress.class=nginx \ + --set ingress.class=webapprouting.kubernetes.azure.com \ --set identity.clientid=$DRONESCHEDULER_CLIENT_ID \ --set identity.serviceAccountName=dronescheduler-sa-v0.1.0 \ --set keyvault.uri=$DRONESCHEDULER_KEYVAULT_URI \ @@ -489,6 +412,11 @@ helm install dronescheduler-v0.1.0-dev dronescheduler-v0.1.0.tgz \ helm status dronescheduler-v0.1.0-dev --namespace backend-dev ``` +## Verify that all the microservice pods are ready and are in running state. +``` +kubectl get pods -n backend-dev --watch +``` + ## Validate the application is running You can send delivery requests and check their statuses using curl.