diff --git a/azuredeploy.bicep b/azuredeploy.bicep index 05a833c4..00c3c13e 100644 --- a/azuredeploy.bicep +++ b/azuredeploy.bicep @@ -22,13 +22,6 @@ param ingestionIdName string @description('Name of the package managed identity') param packageIdName string -@description('Client ID (used by cloudprovider)') -param servicePrincipalClientId string - -@description('The Service Principal Client Secret.') -@secure() -param servicePrincipalClientSecret string - @description('The type of operating system.') @allowed([ 'Linux' @@ -40,30 +33,16 @@ param osType string = 'Linux' @maxValue(1023) param osDiskSizeGB int = 0 -@description('The version of Kubernetes. It must be supported in the target location.') -param kubernetesVersion string +param logAnalyticsWorkspaceID string -@description('Type of the storage account that will store Redis Cache.') -@allowed([ - 'Standard_LRS' - 'Standard_ZRS' - 'Standard_GRS' -]) -param deliveryRedisStorageType string = 'Standard_LRS' - -var clusterNamePrefix = 'aks' var managedIdentityOperatorRoleId = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'f1a07417-d97a-45cb-824c-7a7467783830') -var deliveryRedisStorageName = 'rsto${uniqueString(resourceGroup().id)}' var nestedACRDeploymentName = 'azuredeploy-acr-${acrResourceGroupName}' -var aksLogAnalyticsNamePrefix = 'logsAnalytics' var monitoringMetricsPublisherRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '3913510d-42f4-4e42-8a64-420c390055eb') +var contributorRoleId = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c') var nodeResourceGroupName = 'rg-${aksClusterName}-nodepools' -var aksClusterName = uniqueString(clusterNamePrefix, resourceGroup().id) +var aksClusterName = 'aks-${uniqueString(resourceGroup().id)}' var agentCount = 2 var agentVMSize = 'Standard_D2_v2' -var workspaceName = 'la-${uniqueString(aksLogAnalyticsNamePrefix, resourceGroup().id)}' -var workspaceSku = 'pergb2018' -var workspaceRetentionInDays = 0 module nestedACRDeployment './azuredeploy_nested_nestedACRDeployment.bicep' = { name: nestedACRDeploymentName @@ -74,26 +53,23 @@ module nestedACRDeployment './azuredeploy_nested_nestedACRDeployment.bicep' = { } } -resource workspace 'Microsoft.OperationalInsights/workspaces@2022-10-01' = { - name: workspaceName - location: location - properties: { - retentionInDays: workspaceRetentionInDays - sku: { - name: workspaceSku - } - features: { - searchVersion: 1 - } - } -} - // The control plane identity used by the cluster. Used for networking access (VNET joining and DNS updating) resource miClusterControlPlane 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { name: 'mi-${aksClusterName}-controlplane' location: location } +//provide contributor role to the RG to AKS managed identity. +resource roleAssignment 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = { + name: guid(resourceGroup().id, miClusterControlPlane.id) + scope: resourceGroup() + properties: { + principalId: miClusterControlPlane.properties.principalId + roleDefinitionId: contributorRoleId + principalType: 'ServicePrincipal' + } +} + resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-09-02-preview' = { name: aksClusterName location: location @@ -101,7 +77,6 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-09-02-previ environment: 'shared cluster' } properties: { - kubernetesVersion: kubernetesVersion nodeResourceGroup: nodeResourceGroupName dnsPrefix: aksClusterName agentPoolProfiles: [ @@ -122,14 +97,10 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-09-02-previ mode: 'User' } ] - servicePrincipalProfile: { - clientId: servicePrincipalClientId - secret: servicePrincipalClientSecret - } addonProfiles: { omsagent: { config: { - logAnalyticsWorkspaceResourceID: workspace.id + logAnalyticsWorkspaceResourceID: logAnalyticsWorkspaceID } enabled: true } @@ -174,20 +145,6 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-09-02-previ '${miClusterControlPlane.id}': {} } } - -} - -resource deliveryRedisStorage 'Microsoft.Storage/storageAccounts@2022-09-01' = { - name: deliveryRedisStorageName - sku: { - name: deliveryRedisStorageType - } - kind: 'Storage' - location: location - tags: { - displayName: 'Storage account for inflight deliveries' - app: 'fabrikam-delivery' - } } resource clusterIdentityPublisherRoleAssigment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { diff --git a/charts/package/templates/package-deploy.yaml b/charts/package/templates/package-deploy.yaml index 384cc8f9..ad48d4d0 100644 --- a/charts/package/templates/package-deploy.yaml +++ b/charts/package/templates/package-deploy.yaml @@ -101,6 +101,11 @@ spec: secretKeyRef: name: package-secrets key: appinsights-ikey + - name: APPINSIGHTS_CONNECTION_STRING + valueFrom: + secretKeyRef: + name: package-secrets + key: appinsights-connstr - name: LOG_LEVEL value: {{ .Values.log.level }} - name: CONTAINER_NAME @@ -114,4 +119,4 @@ spec: driver: secrets-store.csi.k8s.io readOnly: true volumeAttributes: - secretProviderClass: package-secrets-csi-akv-{{ $svcversion }} \ No newline at end of file + secretProviderClass: package-secrets-csi-akv-{{ $svcversion }} diff --git a/charts/package/templates/package-secret-provider.yaml b/charts/package/templates/package-secret-provider.yaml index 74302c0a..90d37c39 100644 --- a/charts/package/templates/package-secret-provider.yaml +++ b/charts/package/templates/package-secret-provider.yaml @@ -22,6 +22,8 @@ spec: key: cosmosdb-connstr - objectName: ApplicationInsights--InstrumentationKey key: appinsights-ikey + - objectName: ApplicationInsights--ConnectionString + key: appinsights-connstr parameters: usePodIdentity: "false" clientID: {{ .Values.identity.clientid }} @@ -35,6 +37,10 @@ spec: - | objectName: ApplicationInsights--InstrumentationKey objectAlias: ApplicationInsights--InstrumentationKey - objectType: secret + objectType: secret + - | + objectName: ApplicationInsights--ConnectionString + objectAlias: ApplicationInsights--ConnectionString + objectType: secret tenantId: {{ .Values.identity.tenantId }} --- diff --git a/deployment.md b/deployment.md index 10105917..e27c1076 100644 --- a/deployment.md +++ b/deployment.md @@ -3,117 +3,121 @@ ## Prerequisites - Azure subscription - > Important: The user initiating the deployment process must have access to the **Microsoft.Authorization/roleAssignments/write** permission. For more information, see [the Container Insights doc](https://docs.microsoft.com/azure/azure-monitor/insights/container-insights-troubleshoot#authorization-error-during-onboarding-or-update-operation) + > Important: The user initiating the deployment process must have access to the **Microsoft.Authorization/roleAssignments/write** permission. For more information, see [the Container Insights documentation](https://docs.microsoft.com/azure/azure-monitor/insights/container-insights-troubleshoot#authorization-error-during-onboarding-or-update-operation) - [Azure CLI 2.53.1 or newer](https://docs.microsoft.com/cli/azure/install-azure-cli) -- [Docker](https://docs.docker.com/) - [JQ](https://stedolan.github.io/jq/download/) +- Kubectl + ```bash + az aks install-cli + ``` +- Helm + ```bash + curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash + ``` + +## Clone or download this repo locally. + + ```bash + git clone --recurse-submodules https://github.com/mspnp/microservices-reference-implementation.git && \ + cd microservices-reference-implementation/ + ``` -> Note: in linux systems, it is possible to run the docker command without prefacing -> with sudo. For more information, please refer to [the Post-installation steps -> for linux](https://docs.docker.com/install/linux/linux-postinstall/) +The deployment steps shown here use Bash shell commands. On Windows, you can use the [Windows Subsystem for Linux](https://docs.microsoft.com/windows/wsl/about) to run Bash. -Clone or download this repo locally. +### Log in to Azure CLI ```bash -git clone --recurse-submodules https://github.com/mspnp/microservices-reference-implementation.git && \ -cd microservices-reference-implementation/ +az login ``` -The deployment steps shown here use Bash shell commands. On Windows, you can use the [Windows Subsystem for Linux](https://docs.microsoft.com/windows/wsl/about) to run Bash. +### Set deployment location -## Azure Resources Provisioning + ```bash + export LOCATION=eastus2 + ``` -Set environment variables. +### Deploy the workload's prerequisites ```bash -export LOCATION=eastus2 -``` +az deployment sub create --name workload-stamp-prereqs --location ${LOCATION} --template-file ./workload/workload-stamp-prereqs.bicep --parameters resourceGroupLocation=${LOCATION} -Log in in to Azure. +# Get the workload user assigned identities +DELIVERY_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-delivery --query principalId -o tsv) && \ +DRONESCHEDULER_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-dronescheduler --query principalId -o tsv) && \ +WORKFLOW_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-workflow --query principalId -o tsv) && \ +PACKAGE_ID_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-package --query principalId -o tsv) && \ +INGESTION_ID_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-ingestion --query principalId -o tsv) +``` -```bash -az login +:book: This pre-flight Bicep file creates two resource groups. Additionally five User identities are provisioned that will be later associated to every containerized microservice. This is because they will need Azure RBAC roles over the Azure Key Vault to read secrets at runtime. The resources will be created in the resource group's location, and each resource group will contain the region name as part of its name. -# if you have several subscriptions, select one -# az account set -s -``` +### Deploy the workload related resources -## Deployment +```bash +az deployment group create -f ./workload/workload-stamp.bicep -g rg-shipping-dronedelivery-${LOCATION} -p droneSchedulerPrincipalId=$DRONESCHEDULER_PRINCIPAL_ID -p workflowPrincipalId=$WORKFLOW_PRINCIPAL_ID -p deliveryPrincipalId=$DELIVERY_PRINCIPAL_ID -p ingestionPrincipalId=$INGESTION_ID_PRINCIPAL_ID -p packagePrincipalId=$PACKAGE_ID_PRINCIPAL_ID -> Note: this deployment might take up to 20 minutes +# Assign Azure Container Registry variables +ACR_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.acrName.value -o tsv) +ACR_SERVER=$(az acr show -n $ACR_NAME --query loginServer -o tsv) -Infrastructure +# Assign Log Analytics Workspace variables +export LOG_ANALYTICS_WORKSPACE_ID=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.laWorkspace.value -o tsv) +``` -```bash -# Deploy the managed identities (This takes less than two minutes.) +## Build the microservice images -export PREREQS_DEPLOYMENT_NAME=workload-stamp-prereqs-main +### Steps -az deployment sub create --name $PREREQS_DEPLOYMENT_NAME --location ${LOCATION} --template-file ./workload/workload-stamp-prereqs.bicep --parameters resourceGroupLocation=${LOCATION} +1. Build and push the Delivery service container image to the container registry. -# Get the user identities -export DELIVERY_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-delivery --query principalId -o tsv) && \ -export DRONESCHEDULER_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-dronescheduler --query principalId -o tsv) && \ -export WORKFLOW_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-workflow --query principalId -o tsv) && \ -export PACKAGE_ID_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-package --query principalId -o tsv) && \ -export INGESTION_ID_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-ingestion --query principalId -o tsv) + ```bash + az acr build -r $ACR_NAME -t $ACR_SERVER/delivery:0.1.0 ./workload/src/shipping/delivery/. + ``` +2. Build and push the Ingestion service container image to the container registry. -# Wait for Microsoft Entra ID propagation -until az ad sp show --id $DELIVERY_PRINCIPAL_ID &> /dev/null ; do echo "Waiting for Microsoft Entra ID propagation" && sleep 5; done -until az ad sp show --id $DRONESCHEDULER_PRINCIPAL_ID &> /dev/null ; do echo "Waiting for Microsoft Entra ID propagation" && sleep 5; done -until az ad sp show --id $WORKFLOW_PRINCIPAL_ID &> /dev/null ; do echo "Waiting for Microsoft Entra ID propagation" && sleep 5; done -until az ad sp show --id $PACKAGE_ID_PRINCIPAL_ID &> /dev/null ; do echo "Waiting for Microsoft Entra ID propagation" && sleep 5; done -until az ad sp show --id $INGESTION_ID_PRINCIPAL_ID &> /dev/null ; do echo "Waiting for Microsoft Entra ID propagation" && sleep 5; done + ```bash + az acr build -r $ACR_NAME -t $ACR_SERVER/ingestion:0.1.0 ./workload/src/shipping/ingestion/. + ``` -# Deploy all the workload related resources (This step takes about 10 minutes) -az deployment group create -f ./workload/workload-stamp.bicep -g rg-shipping-dronedelivery-${LOCATION} -p droneSchedulerPrincipalId=$DRONESCHEDULER_PRINCIPAL_ID -p workflowPrincipalId=$WORKFLOW_PRINCIPAL_ID -p deliveryPrincipalId=$DELIVERY_PRINCIPAL_ID -p ingestionPrincipalId=$INGESTION_ID_PRINCIPAL_ID -p packagePrincipalId=$PACKAGE_ID_PRINCIPAL_ID +3. Build and push the Workflow service container image to the container registry. -# Get outputs from workload deploy -export ACR_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.acrName.value -o tsv) -export ACR_SERVER=$(az acr show -n $ACR_NAME --query loginServer -o tsv) -``` + ```bash + az acr build -r $ACR_NAME -t $ACR_SERVER/workflow:0.1.0 ./workload/src/shipping/workflow/. + ``` -Deploy the managed cluster and all related resources (This step takes about 15 minutes) +4. Build and push the DroneScheduler service container image to the container registry. -```bash -export RESOURCE_GROUP_ID=$(az group show --name rg-shipping-dronedelivery-${LOCATION} --query id --output tsv) + ```bash + az acr build -r $ACR_NAME -f ./workload/src/shipping/dronescheduler/Dockerfile -t $ACR_SERVER/dronescheduler:0.1.0 ./workload/src/shipping/. + ``` -export SP_DETAILS=$(az ad sp create-for-rbac --role="Contributor" --scopes $RESOURCE_GROUP_ID -o json) && \ -export SP_APP_ID=$(echo $SP_DETAILS | jq ".appId" -r) && \ -export SP_CLIENT_SECRET=$(echo $SP_DETAILS | jq ".password" -r) -export TENANT_ID=$(az account show --query tenantId --output tsv) +5. Build and push the Package service container image to the container registry. -export DEPLOYMENT_SUFFIX=$(date +%S%N) + ```bash + az acr build -r $ACR_NAME -t $ACR_SERVER/package:0.1.0 ./workload/src/shipping/package/. + ``` -export KUBERNETES_VERSION=$(az aks get-versions -l $LOCATION --query "values[?isDefault].version" -o tsv) +## Deploy the managed cluster and related resources -export DEPLOYMENT_NAME=azuredeploy-$DEPLOYMENT_SUFFIX -az deployment group create -g rg-shipping-dronedelivery-${LOCATION} --name $DEPLOYMENT_NAME --template-file azuredeploy.bicep \ ---parameters servicePrincipalClientId=$SP_APP_ID \ - servicePrincipalClientSecret=$SP_CLIENT_SECRET \ - kubernetesVersion=$KUBERNETES_VERSION \ +```bash +az deployment group create -g rg-shipping-dronedelivery-${LOCATION} --name managed-cluster-deployment --template-file azuredeploy.bicep --parameters \ deliveryIdName=uid-delivery \ ingestionIdName=uid-ingestion \ packageIdName=uid-package \ droneSchedulerIdName=uid-dronescheduler \ workflowIdName=uid-workflow \ acrResourceGroupName=rg-shipping-dronedelivery-${LOCATION}-acr \ - acrName=$ACR_NAME -``` + acrName=$ACR_NAME \ + logAnalyticsWorkspaceID=$LOG_ANALYTICS_WORKSPACE_ID -Get the cluster name output from Azure Deploy. - -```bash -export CLUSTER_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n $DEPLOYMENT_NAME --query properties.outputs.aksClusterName.value -o tsv) -echo $CLUSTER_NAME +# Get the AKS cluster name from the Azure deployment output +export CLUSTER_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n managed-cluster-deployment --query properties.outputs.aksClusterName.value -o tsv) ``` -Download kubectl and create a Kubernetes namespace. +### Get the AKS cluster credentials and create a Kubernetes namespace ```bash -# Install kubectl -az aks install-cli # Get the Kubernetes cluster credentials az aks get-credentials --resource-group=rg-shipping-dronedelivery-${LOCATION} --name=$CLUSTER_NAME @@ -122,55 +126,60 @@ az aks get-credentials --resource-group=rg-shipping-dronedelivery-${LOCATION} -- kubectl create namespace backend-dev ``` -Install and initialize Helm. - -```bash -# install helm 3 -curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash -``` - -Integrate Application Insights instance. +### Configure RBAC permissions for Azure Application Insights ```bash -# Acquire Instrumentation Key -export AI_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.appInsightsName.value -o tsv) -echo $AI_NAME # add RBAC for AppInsights kubectl apply -f k8s/k8s-rbac-ai.yaml ``` -## Verify that the secrets-store pods are running in the kube-system namespace +### Set resource quotas for the namespace ```bash -kubectl get pods -n kube-system +kubectl apply -f k8s/k8s-resource-quotas-dev.yaml ``` -You should see an output similar to this: +### Get the cluster OpenID Connect issuer URL, and the tenant ID and object ID of the signed-in user ```bash -NAME READY STATUS RESTARTS AGE -aks-secrets-store-csi-driver-4bjzx 3/3 Running 2 28m -aks-secrets-store-csi-driver-b22bj 3/3 Running 1 28m -aks-secrets-store-provider-azure-2k5mx 1/1 Running 0 28m -aks-secrets-store-provider-azure-l5w98 1/1 Running 0 28m +export AKS_OIDC_ISSUER="$(az aks show -n $CLUSTER_NAME -g rg-shipping-dronedelivery-${LOCATION} --query "oidcIssuerProfile.issuerUrl" -otsv)" +export TENANT_ID=$(az account show --query tenantId --output tsv) +export SIGNED_IN_OBJECT_ID=$(az ad signed-in-user show --query 'id' -o tsv) ``` -## Collect details of managed ingress controller. +### Create managed identity federations for microservices +``` +# Set up the Delivery microservice's managed identity to trust your Kubernetes service account. +az identity federated-credential create --name credential-for-delivery --identity-name uid-delivery --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:delivery-sa-v0.1.0 +# Set up the Package microservice's managed identity to trust your Kubernetes service account. +az identity federated-credential create --name credential-for-package --identity-name uid-package --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:package-sa-v0.1.0 + +# Set up the Workflow microservice's managed identity to trust your Kubernetes service account. +az identity federated-credential create --name credential-for-workflow --identity-name uid-workflow --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:workflow-sa-v0.1.0 + +# Set up the Ingestion microservice's managed identity to trust your Kubernetes service account. +az identity federated-credential create --name credential-for-ingestion --identity-name uid-ingestion --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:ingestion-sa-v0.1.0 + +# Set up the Drone Scheduler microservice's managed identity to trust your Kubernetes service account. +az identity federated-credential create --name credential-for-dronescheduler --identity-name uid-dronescheduler --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:dronescheduler-sa-v0.1.0 + +``` + +### Collect details of the managed ingress controller ```bash # Obtain the load balancer ip address of managed ingress and assign a domain name export INGRESS_LOAD_BALANCER_IP=$(kubectl get service -n app-routing-system nginx -o jsonpath="{.status.loadBalancer.ingress[0].ip}" 2> /dev/null) - export INGRESS_LOAD_BALANCER_IP_ID=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$INGRESS_LOAD_BALANCER_IP')].[id]" --output tsv) && \ export EXTERNAL_INGEST_DNS_NAME="dronedelivery-${LOCATION}-${RANDOM}-ing" && \ export EXTERNAL_INGEST_FQDN=$(az network public-ip update --ids $INGRESS_LOAD_BALANCER_IP_ID --dns-name $EXTERNAL_INGEST_DNS_NAME --query "dnsSettings.fqdn" --output tsv) ``` -## Create self-signed certificate for TLS +### Create self-signed certificate for TLS > :warning: WARNING > @@ -179,7 +188,6 @@ export EXTERNAL_INGEST_FQDN=$(az network public-ip update --ids $INGRESS_LOAD_BA > For your production cluster, use your > security best practices for digital certificates creation and lifetime management. - ```bash # Create a self-signed certificate for TLS @@ -189,22 +197,8 @@ openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ -subj "/CN=${EXTERNAL_INGEST_FQDN}/O=fabrikam" ``` -## Setup cluster resource quota - -```bash -kubectl apply -f k8s/k8s-resource-quotas-dev.yaml -``` - -## Get the OIDC Issuer URL - -```bash -export AKS_OIDC_ISSUER="$(az aks show -n $CLUSTER_NAME -g rg-shipping-dronedelivery-${LOCATION} --query "oidcIssuerProfile.issuerUrl" -otsv)" -``` - ## Deploy the Delivery service -Extract resource details from deployment. - ```bash export COSMOSDB_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.deliveryCosmosDbName.value -o tsv) && \ export DATABASE_NAME="${COSMOSDB_NAME}-db" && \ @@ -212,49 +206,35 @@ export COLLECTION_NAME="${DATABASE_NAME}-col" && \ export DELIVERY_KEYVAULT_URI=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.deliveryKeyVaultUri.value -o tsv) && \ export DELIVERY_KEYVAULT_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.deliveryKeyVaultName.value -o tsv) && \ export DELIVERY_PRINCIPAL_CLIENT_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-delivery --query clientId -o tsv) -``` - -Build and publish the Delivery service container image. - -```bash -az acr build -r $ACR_NAME -t $ACR_SERVER/delivery:0.1.0 ./workload/src/shipping/delivery/. -``` - -Deploy the Delivery service. +export DELIVERY_KEYVAULT_ID=$(az resource show -g rg-shipping-dronedelivery-${LOCATION} -n $DELIVERY_KEYVAULT_NAME --resource-type 'Microsoft.KeyVault/vaults' --query id --output tsv) -```bash # Create secrets -# Note: Ingress TLS key and certificate secrets cannot be exported as outputs in ARM deployments # The current user is given permission to import secrets and then it is deleted right after the secret creation command is executed -export SIGNED_IN_OBJECT_ID=$(az ad signed-in-user show --query 'id' -o tsv) -export DELIVERY_KEYVAULT_ID=$(az resource show -g rg-shipping-dronedelivery-${LOCATION} -n $DELIVERY_KEYVAULT_NAME --resource-type 'Microsoft.KeyVault/vaults' --query id --output tsv) az role assignment create --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $DELIVERY_KEYVAULT_ID +# Wait for the role assignment to finish propagating. +sleep 30 az keyvault secret set --name Delivery-Ingress-Tls-Key --vault-name $DELIVERY_KEYVAULT_NAME --value "$(cat ingestion-ingress-tls.key)" az keyvault secret set --name Delivery-Ingress-Tls-Crt --vault-name $DELIVERY_KEYVAULT_NAME --value "$(cat ingestion-ingress-tls.crt)" - az role assignment delete --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $DELIVERY_KEYVAULT_ID -#Setup your managed identity to trust your Kubernetes service account -az identity federated-credential create --name credential-for-delivery --identity-name uid-delivery --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:delivery-sa-v0.1.0 - # Deploy the service helm package charts/delivery/ -u && \ helm install delivery-v0.1.0-dev delivery-v0.1.0.tgz \ --set image.tag=0.1.0 \ --set image.repository=delivery \ --set dockerregistry=$ACR_SERVER \ - --set ingress.hosts[0].name=$EXTERNAL_INGEST_FQDN \ - --set ingress.hosts[0].serviceName=delivery \ - --set ingress.hosts[0].tls=true \ - --set ingress.hosts[0].tlsSecretName=delivery-ingress-tls \ + --set ingress.hosts\[0\].name=$EXTERNAL_INGEST_FQDN \ + --set ingress.hosts\[0\].serviceName=delivery \ + --set ingress.hosts\[0\].tls=true \ + --set ingress.hosts\[0\].tlsSecretName=delivery-ingress-tls \ --set identity.clientid=$DELIVERY_PRINCIPAL_CLIENT_ID \ --set identity.serviceAccountName=delivery-sa-v0.1.0 \ --set identity.tenantId=$TENANT_ID \ --set keyVaultName=$DELIVERY_KEYVAULT_NAME \ --set ingress.tls=true \ - --set ingress.class=nginx \ + --set ingress.class=webapprouting.kubernetes.azure.com \ --set cosmosdb.id=$DATABASE_NAME \ --set cosmosdb.collectionid=$COLLECTION_NAME \ --set keyvault.uri=$DELIVERY_KEYVAULT_URI \ @@ -263,45 +243,20 @@ helm install delivery-v0.1.0-dev delivery-v0.1.0.tgz \ --namespace backend-dev \ --dependency-update -# Verify the pod is created +# Verify the Helm deployment status helm status delivery-v0.1.0-dev --namespace backend-dev + +# Verify that the delivery microservice pod is in running state (Ctrl+C to quit) +kubectl get pods -n backend-dev --watch ``` ## Deploy the Package service -Extract resource details from deployment. - ```bash -export COSMOSDB_NAME_PACKAGE=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.packageMongoDbName.value -o tsv) export PACKAGE_KEYVAULT_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.packageKeyVaultName.value -o tsv) export PACKAGE_ID_CLIENT_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-package --query clientId -o tsv) -``` - -Build the Package service. - -```bash -az acr build -r $ACR_NAME -t $ACR_SERVER/package:0.1.0 ./workload/src/shipping/package/. -``` - -Deploy the Package service. - -```bash -# Create secret -# Note: Connection strings cannot be exported as outputs in ARM deployments -# The current user is given permission to import secrets and then it is deleted right after the secret creation command is executed -export COSMOSDB_CONNECTION_PACKAGE=$(az cosmosdb keys list --type connection-strings --name $COSMOSDB_NAME_PACKAGE --resource-group rg-shipping-dronedelivery-${LOCATION} --query "connectionStrings[0].connectionString" -o tsv | sed 's/==/%3D%3D/g') && \ export COSMOSDB_COL_NAME_PACKAGE=packages -export PACKAGE_KEYVAULT_ID=$(az resource show -g rg-shipping-dronedelivery-${LOCATION} -n $PACKAGE_KEYVAULT_NAME --resource-type 'Microsoft.KeyVault/vaults' --query id --output tsv) -az role assignment create --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $PACKAGE_KEYVAULT_ID - -az keyvault secret set --name CosmosDb--ConnectionString --vault-name $PACKAGE_KEYVAULT_NAME --value $COSMOSDB_CONNECTION_PACKAGE - -az role assignment delete --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $PACKAGE_KEYVAULT_ID - -# Setup your managed identity to trust your Kubernetes service account -az identity federated-credential create --name credential-for-package --identity-name uid-package --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:package-sa-v0.1.0 - # Deploy service helm package charts/package/ -u && \ helm install package-v0.1.0-dev package-v0.1.0.tgz \ @@ -311,10 +266,10 @@ helm install package-v0.1.0-dev package-v0.1.0.tgz \ --set identity.serviceAccountName=package-sa-v0.1.0 \ --set identity.tenantId=$TENANT_ID \ --set keyVaultName=$PACKAGE_KEYVAULT_NAME \ - --set ingress.hosts[0].name=$EXTERNAL_INGEST_FQDN \ - --set ingress.hosts[0].serviceName=package \ - --set ingress.hosts[0].tls=false \ - --set ingress.class=nginx \ + --set ingress.hosts\[0\].name=$EXTERNAL_INGEST_FQDN \ + --set ingress.hosts\[0\].serviceName=package \ + --set ingress.hosts\[0\].tls=false \ + --set ingress.class=webapprouting.kubernetes.azure.com \ --set cosmosDb.collectionName=$COSMOSDB_COL_NAME_PACKAGE \ --set dockerregistry=$ACR_SERVER \ --set reason="Initial deployment" \ @@ -322,14 +277,15 @@ helm install package-v0.1.0-dev package-v0.1.0.tgz \ --namespace backend-dev \ --dependency-update -# Verify the pod is created +# Verify the Helm deployment status helm status package-v0.1.0-dev --namespace backend-dev + +# Verify that the package microservice pod is in running state (Ctrl+C to quit) +kubectl get pods -n backend-dev --watch ``` ## Deploy the Workflow service -Extract resource details from deployment. - ```bash export WORKFLOW_KEYVAULT_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.workflowKeyVaultName.value -o tsv) export WORKFLOW_ID_CLIENT_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-workflow --query clientId -o tsv) @@ -338,20 +294,6 @@ export WORKFLOW_NAMESPACE_NAME=$(az deployment group show -g rg-shipping-dronede export WORKFLOW_NAMESPACE_ENDPOINT=$(az servicebus namespace show -g rg-shipping-dronedelivery-${LOCATION} -n $WORKFLOW_NAMESPACE_NAME --query serviceBusEndpoint -o tsv) export WORKFLOW_NAMESPACE_SAS_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.workflowServiceAccessKeyName.value -o tsv) -``` - -Build the workflow service. - -```bash -az acr build -r $ACR_NAME -t $ACR_SERVER/workflow:0.1.0 ./workload/src/shipping/workflow/. -``` - -Deploy the Workflow service. - -```bash -# Setup your managed identity to trust your Kubernetes service account -az identity federated-credential create --name credential-for-workflow --identity-name uid-workflow --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:workflow-sa-v0.1.0 - # Deploy the service helm package charts/workflow/ -u && \ helm install workflow-v0.1.0-dev workflow-v0.1.0.tgz \ @@ -374,41 +316,31 @@ helm install workflow-v0.1.0-dev workflow-v0.1.0.tgz \ --namespace backend-dev \ --dependency-update -# Verify the pod is created +# Verify the Helm deployment status helm status workflow-v0.1.0-dev --namespace backend-dev + +# Verify that the workflow microservice pod is in running state (Ctrl+C to quit) +kubectl get pods -n backend-dev --watch ``` ## Deploy the Ingestion service -Extract resource details and pod identity outputs from deployment. - ```bash export INGESTION_QUEUE_NAMESPACE=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.ingestionQueueNamespace.value -o tsv) && \ export INGESTION_QUEUE_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.ingestionQueueName.value -o tsv) export INGESTION_KEYVAULT_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.ingestionKeyVaultName.value -o tsv) export INGESTION_ID_CLIENT_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-ingestion --query clientId -o tsv) +export INGESTION_KEYVAULT_ID=$(az resource show -g rg-shipping-dronedelivery-${LOCATION} -n $INGESTION_KEYVAULT_NAME --resource-type 'Microsoft.KeyVault/vaults' --query id --output tsv) # The current user is given permission to import secrets and then it is deleted right after the secret creation command is executed -export INGESTION_KEYVAULT_ID=$(az resource show -g rg-shipping-dronedelivery-${LOCATION} -n $INGESTION_KEYVAULT_NAME --resource-type 'Microsoft.KeyVault/vaults' --query id --output tsv) az role assignment create --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $INGESTION_KEYVAULT_ID +# Wait for the role assignment to finish propagating. +sleep 30 az keyvault secret set --name Ingestion-Ingress-Tls-Key --vault-name $INGESTION_KEYVAULT_NAME --value "$(cat ingestion-ingress-tls.key)" az keyvault secret set --name Ingestion-Ingress-Tls-Crt --vault-name $INGESTION_KEYVAULT_NAME --value "$(cat ingestion-ingress-tls.crt)" az role assignment delete --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $INGESTION_KEYVAULT_ID -``` - -Build the Ingestion service. - -```bash -az acr build -r $ACR_NAME -t $ACR_SERVER/ingestion:0.1.0 ./workload/src/shipping/ingestion/. -``` - -Deploy the Ingestion service - -```bash -# Setup your managed identity to trust your Kubernetes service account -az identity federated-credential create --name credential-for-ingestion --identity-name uid-ingestion --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:ingestion-sa-v0.1.0 # Deploy service helm package charts/ingestion/ -u && \ @@ -420,12 +352,12 @@ helm install ingestion-v0.1.0-dev ingestion-v0.1.0.tgz \ --set identity.serviceAccountName=ingestion-sa-v0.1.0 \ --set identity.tenantId=$TENANT_ID \ --set keyVaultName=$INGESTION_KEYVAULT_NAME \ - --set ingress.hosts[0].name=$EXTERNAL_INGEST_FQDN \ - --set ingress.hosts[0].serviceName=ingestion \ - --set ingress.hosts[0].tls=true \ - --set ingress.hosts[0].tlsSecretName=ingestion-ingress-tls \ + --set ingress.hosts\[0\].name=$EXTERNAL_INGEST_FQDN \ + --set ingress.hosts\[0\].serviceName=ingestion \ + --set ingress.hosts\[0\].tls=true \ + --set ingress.hosts\[0\].tlsSecretName=ingestion-ingress-tls \ --set ingress.tls=true \ - --set ingress.class=nginx \ + --set ingress.class=webapprouting.kubernetes.azure.com \ --set secrets.queue.keyname=IngestionServiceAccessKey \ --set secrets.queue.name=${INGESTION_QUEUE_NAME} \ --set secrets.queue.namespace=${INGESTION_QUEUE_NAMESPACE} \ @@ -434,14 +366,15 @@ helm install ingestion-v0.1.0-dev ingestion-v0.1.0.tgz \ --namespace backend-dev \ --dependency-update -# Verify the pod is created +# Verify the Helm deployment status helm status ingestion-v0.1.0-dev --namespace backend-dev + +# Verify that the ingestion microservice pod is in running state (Ctrl+C to quit) +kubectl get pods -n backend-dev --watch ``` ## Deploy DroneScheduler service -Extract resource details from deployment. - ```bash export DRONESCHEDULER_KEYVAULT_URI=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.droneSchedulerKeyVaultUri.value -o tsv) export DRONESCHEDULER_COSMOSDB_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.droneSchedulerCosmosDbName.value -o tsv) && \ @@ -450,19 +383,6 @@ export AUTH_KEY=$(az cosmosdb keys list -n $DRONESCHEDULER_COSMOSDB_NAME -g rg-s export DRONESCHEDULER_CLIENT_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-dronescheduler --query clientId -o tsv) && \ export DATABASE_NAME="invoicing" && \ export COLLECTION_NAME="utilization" -``` - -Build and publish the container image. - -```bash -az acr build -r $ACR_NAME -f ./workload/src/shipping/dronescheduler/Dockerfile -t $ACR_SERVER/dronescheduler:0.1.0 ./workload/src/shipping/. -``` - -Deploy the dronescheduler service. - -```bash -# Setup your managed identity to trust your Kubernetes service account -az identity federated-credential create --name credential-for-dronescheduler --identity-name uid-dronescheduler --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:dronescheduler-sa-v0.1.0 # Deploy the service helm package charts/dronescheduler/ -u && \ @@ -470,10 +390,10 @@ helm install dronescheduler-v0.1.0-dev dronescheduler-v0.1.0.tgz \ --set image.tag=0.1.0 \ --set image.repository=dronescheduler \ --set dockerregistry=$ACR_SERVER \ - --set ingress.hosts[0].name=$EXTERNAL_INGEST_FQDN \ - --set ingress.hosts[0].serviceName=dronescheduler \ - --set ingress.hosts[0].tls=false \ - --set ingress.class=nginx \ + --set ingress.hosts\[0\].name=$EXTERNAL_INGEST_FQDN \ + --set ingress.hosts\[0\].serviceName=dronescheduler \ + --set ingress.hosts\[0\].tls=false \ + --set ingress.class=webapprouting.kubernetes.azure.com \ --set identity.clientid=$DRONESCHEDULER_CLIENT_ID \ --set identity.serviceAccountName=dronescheduler-sa-v0.1.0 \ --set keyvault.uri=$DRONESCHEDULER_KEYVAULT_URI \ @@ -485,8 +405,17 @@ helm install dronescheduler-v0.1.0-dev dronescheduler-v0.1.0.tgz \ --namespace backend-dev \ --dependency-update -# Verify the pod is created +# Verify the Helm deployment status helm status dronescheduler-v0.1.0-dev --namespace backend-dev + +# Verify that the drone scheduler microservice pod is in running state (Ctrl+C to quit) +kubectl get pods -n backend-dev --watch +``` + +## Verify that all the microservice pods are ready and are in the running state, and that the ingress routes are deployed. +``` +kubectl get pods -n backend-dev +kubectl get ing -n backend-dev ``` ## Validate the application is running diff --git a/workload b/workload index 820aca14..49104280 160000 --- a/workload +++ b/workload @@ -1 +1 @@ -Subproject commit 820aca14374d56bdf062f8d08d72591756d00903 +Subproject commit 491042808a70463dddeb6dc91d2b49fecd2150e0