From b2c63252432a592234cca7cc42e0a83b2f8b7e8d Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Mon, 27 Jan 2025 09:47:20 +0300 Subject: [PATCH 01/44] adding app insights connection string as a kubernetes secret --- charts/package/templates/package-secret-provider.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/charts/package/templates/package-secret-provider.yaml b/charts/package/templates/package-secret-provider.yaml index 74302c0a..90d37c39 100644 --- a/charts/package/templates/package-secret-provider.yaml +++ b/charts/package/templates/package-secret-provider.yaml @@ -22,6 +22,8 @@ spec: key: cosmosdb-connstr - objectName: ApplicationInsights--InstrumentationKey key: appinsights-ikey + - objectName: ApplicationInsights--ConnectionString + key: appinsights-connstr parameters: usePodIdentity: "false" clientID: {{ .Values.identity.clientid }} @@ -35,6 +37,10 @@ spec: - | objectName: ApplicationInsights--InstrumentationKey objectAlias: ApplicationInsights--InstrumentationKey - objectType: secret + objectType: secret + - | + objectName: ApplicationInsights--ConnectionString + objectAlias: ApplicationInsights--ConnectionString + objectType: secret tenantId: {{ .Values.identity.tenantId }} --- From 62921ab5a2ba454aee67fd41f8662c4eaeed1169 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Mon, 27 Jan 2025 09:50:05 +0300 Subject: [PATCH 02/44] Update package-deploy.yaml --- charts/package/templates/package-deploy.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/charts/package/templates/package-deploy.yaml b/charts/package/templates/package-deploy.yaml index 384cc8f9..ad48d4d0 100644 --- a/charts/package/templates/package-deploy.yaml +++ b/charts/package/templates/package-deploy.yaml @@ -101,6 +101,11 @@ spec: secretKeyRef: name: package-secrets key: appinsights-ikey + - name: APPINSIGHTS_CONNECTION_STRING + valueFrom: + secretKeyRef: + name: package-secrets + key: appinsights-connstr - name: LOG_LEVEL value: {{ .Values.log.level }} - name: CONTAINER_NAME @@ -114,4 +119,4 @@ spec: driver: secrets-store.csi.k8s.io readOnly: true volumeAttributes: - secretProviderClass: package-secrets-csi-akv-{{ $svcversion }} \ No newline at end of file + secretProviderClass: package-secrets-csi-akv-{{ $svcversion }} From 92c108885d6a0811e49025686ddcf35c671efa58 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Wed, 29 Jan 2025 11:40:37 +0300 Subject: [PATCH 03/44] Update deployment.md --- deployment.md | 84 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 57 insertions(+), 27 deletions(-) diff --git a/deployment.md b/deployment.md index 10105917..4d11c2a6 100644 --- a/deployment.md +++ b/deployment.md @@ -21,7 +21,7 @@ cd microservices-reference-implementation/ The deployment steps shown here use Bash shell commands. On Windows, you can use the [Windows Subsystem for Linux](https://docs.microsoft.com/windows/wsl/about) to run Bash. -## Azure Resources Provisioning +## Deploy an Azure Container Registry (ACR) Set environment variables. @@ -29,49 +29,79 @@ Set environment variables. export LOCATION=eastus2 ``` -Log in in to Azure. +### Log in to Azure CLI ```bash az login +``` + +### Deploy the workload's prerequisites -# if you have several subscriptions, select one -# az account set -s +```bash +az deployment sub create --name workload-stamp-prereqs --location ${LOCATION} --template-file ./workload-stamp-prereqs.bicep ``` -## Deployment +:book: This pre-flight Bicep template is creating a general purpose resource group as well as one dedicated for the Azure Container Registry. Additionally five User Identites are provisioned as part of this too that will be later associated to every containerized microservice. This is because they will need Azure RBAC roles over the Azure KeyVault to read secrets in runtime. The resources will be created on the resouce group location and each resource group will contain the region as part of their names -> Note: this deployment might take up to 20 minutes +### Get the workload user assigned identities -Infrastructure +```bash +DELIVERY_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-delivery --query principalId -o tsv) && \ +DRONESCHEDULER_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-dronescheduler --query principalId -o tsv) && \ +WORKFLOW_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-workflow --query principalId -o tsv) && \ +PACKAGE_ID_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-package --query principalId -o tsv) && \ +INGESTION_ID_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-ingestion --query principalId -o tsv) +``` + +### Deploy the workload ```bash -# Deploy the managed identities (This takes less than two minutes.) +az deployment group create -f ./workload-stamp.bicep -g rg-shipping-dronedelivery-${LOCATION} -p droneSchedulerPrincipalId=$DRONESCHEDULER_PRINCIPAL_ID \ +-p workflowPrincipalId=$WORKFLOW_PRINCIPAL_ID \ +-p deliveryPrincipalId=$DELIVERY_PRINCIPAL_ID \ +-p ingestionPrincipalId=$INGESTION_ID_PRINCIPAL_ID \ +-p packagePrincipalId=$PACKAGE_ID_PRINCIPAL_ID +``` -export PREREQS_DEPLOYMENT_NAME=workload-stamp-prereqs-main +### Assign ACR variables -az deployment sub create --name $PREREQS_DEPLOYMENT_NAME --location ${LOCATION} --template-file ./workload/workload-stamp-prereqs.bicep --parameters resourceGroupLocation=${LOCATION} +```bash +ACR_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.acrName.value -o tsv) +ACR_SERVER=$(az acr show -n $ACR_NAME --query loginServer -o tsv) +``` + +## Build the microservice images -# Get the user identities -export DELIVERY_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-delivery --query principalId -o tsv) && \ -export DRONESCHEDULER_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-dronescheduler --query principalId -o tsv) && \ -export WORKFLOW_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-workflow --query principalId -o tsv) && \ -export PACKAGE_ID_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-package --query principalId -o tsv) && \ -export INGESTION_ID_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-ingestion --query principalId -o tsv) +### Steps +1. Build the Delivery service. -# Wait for Microsoft Entra ID propagation -until az ad sp show --id $DELIVERY_PRINCIPAL_ID &> /dev/null ; do echo "Waiting for Microsoft Entra ID propagation" && sleep 5; done -until az ad sp show --id $DRONESCHEDULER_PRINCIPAL_ID &> /dev/null ; do echo "Waiting for Microsoft Entra ID propagation" && sleep 5; done -until az ad sp show --id $WORKFLOW_PRINCIPAL_ID &> /dev/null ; do echo "Waiting for Microsoft Entra ID propagation" && sleep 5; done -until az ad sp show --id $PACKAGE_ID_PRINCIPAL_ID &> /dev/null ; do echo "Waiting for Microsoft Entra ID propagation" && sleep 5; done -until az ad sp show --id $INGESTION_ID_PRINCIPAL_ID &> /dev/null ; do echo "Waiting for Microsoft Entra ID propagation" && sleep 5; done +```bash +az acr build -r $ACR_NAME -t $ACR_SERVER/delivery:0.1.0 ./src/shipping/delivery/. +``` -# Deploy all the workload related resources (This step takes about 10 minutes) -az deployment group create -f ./workload/workload-stamp.bicep -g rg-shipping-dronedelivery-${LOCATION} -p droneSchedulerPrincipalId=$DRONESCHEDULER_PRINCIPAL_ID -p workflowPrincipalId=$WORKFLOW_PRINCIPAL_ID -p deliveryPrincipalId=$DELIVERY_PRINCIPAL_ID -p ingestionPrincipalId=$INGESTION_ID_PRINCIPAL_ID -p packagePrincipalId=$PACKAGE_ID_PRINCIPAL_ID +2. Build the Ingestion service. -# Get outputs from workload deploy -export ACR_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.acrName.value -o tsv) -export ACR_SERVER=$(az acr show -n $ACR_NAME --query loginServer -o tsv) +```bash +az acr build -r $ACR_NAME -t $ACR_SERVER/ingestion:0.1.0 ./src/shipping/ingestion/. +``` + +3. Build the Workflow service. + +```bash +az acr build -r $ACR_NAME -t $ACR_SERVER/workflow:0.1.0 ./src/shipping/workflow/. +``` + +4. Build the DroneScheduler service. + +```bash +az acr build -r $ACR_NAME -f ./src/shipping/dronescheduler/Dockerfile -t $ACR_SERVER/dronescheduler:0.1.0 ./src/shipping/. +``` + +5. Build the Package service. + +```bash +az acr build -r $ACR_NAME -t $ACR_SERVER/package:0.1.0 ./src/shipping/package/. ``` Deploy the managed cluster and all related resources (This step takes about 15 minutes) From 9cba27d16528bfe1c5efaef600f200bccce1b985 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Wed, 29 Jan 2025 11:43:19 +0300 Subject: [PATCH 04/44] Update deployment.md --- deployment.md | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/deployment.md b/deployment.md index 4d11c2a6..14e6217f 100644 --- a/deployment.md +++ b/deployment.md @@ -244,12 +244,6 @@ export DELIVERY_KEYVAULT_NAME=$(az deployment group show -g rg-shipping-dronedel export DELIVERY_PRINCIPAL_CLIENT_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-delivery --query clientId -o tsv) ``` -Build and publish the Delivery service container image. - -```bash -az acr build -r $ACR_NAME -t $ACR_SERVER/delivery:0.1.0 ./workload/src/shipping/delivery/. -``` - Deploy the Delivery service. ```bash @@ -307,12 +301,6 @@ export PACKAGE_KEYVAULT_NAME=$(az deployment group show -g rg-shipping-dronedeli export PACKAGE_ID_CLIENT_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-package --query clientId -o tsv) ``` -Build the Package service. - -```bash -az acr build -r $ACR_NAME -t $ACR_SERVER/package:0.1.0 ./workload/src/shipping/package/. -``` - Deploy the Package service. ```bash @@ -370,12 +358,6 @@ export WORKFLOW_NAMESPACE_SAS_NAME=$(az deployment group show -g rg-shipping-dro ``` -Build the workflow service. - -```bash -az acr build -r $ACR_NAME -t $ACR_SERVER/workflow:0.1.0 ./workload/src/shipping/workflow/. -``` - Deploy the Workflow service. ```bash @@ -428,12 +410,6 @@ az keyvault secret set --name Ingestion-Ingress-Tls-Crt --vault-name $INGESTION_ az role assignment delete --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $INGESTION_KEYVAULT_ID ``` -Build the Ingestion service. - -```bash -az acr build -r $ACR_NAME -t $ACR_SERVER/ingestion:0.1.0 ./workload/src/shipping/ingestion/. -``` - Deploy the Ingestion service ```bash @@ -482,12 +458,6 @@ export DATABASE_NAME="invoicing" && \ export COLLECTION_NAME="utilization" ``` -Build and publish the container image. - -```bash -az acr build -r $ACR_NAME -f ./workload/src/shipping/dronescheduler/Dockerfile -t $ACR_SERVER/dronescheduler:0.1.0 ./workload/src/shipping/. -``` - Deploy the dronescheduler service. ```bash From ee4932512e338a5e62e01cba73d9bbb05e38ed1a Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Wed, 29 Jan 2025 11:45:51 +0300 Subject: [PATCH 05/44] Update deployment.md --- deployment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment.md b/deployment.md index 14e6217f..d18f30de 100644 --- a/deployment.md +++ b/deployment.md @@ -104,7 +104,7 @@ az acr build -r $ACR_NAME -f ./src/shipping/dronescheduler/Dockerfile -t $ACR_SE az acr build -r $ACR_NAME -t $ACR_SERVER/package:0.1.0 ./src/shipping/package/. ``` -Deploy the managed cluster and all related resources (This step takes about 15 minutes) +## Deploy the managed cluster and all related resources (This step takes about 15 minutes) ```bash export RESOURCE_GROUP_ID=$(az group show --name rg-shipping-dronedelivery-${LOCATION} --query id --output tsv) From c734066945804aefb1f94762abfa7c63894c0083 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Wed, 29 Jan 2025 13:13:06 +0300 Subject: [PATCH 06/44] Update deployment.md --- deployment.md | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/deployment.md b/deployment.md index d18f30de..6c091d80 100644 --- a/deployment.md +++ b/deployment.md @@ -5,12 +5,17 @@ - Azure subscription > Important: The user initiating the deployment process must have access to the **Microsoft.Authorization/roleAssignments/write** permission. For more information, see [the Container Insights doc](https://docs.microsoft.com/azure/azure-monitor/insights/container-insights-troubleshoot#authorization-error-during-onboarding-or-update-operation) - [Azure CLI 2.53.1 or newer](https://docs.microsoft.com/cli/azure/install-azure-cli) -- [Docker](https://docs.docker.com/) - [JQ](https://stedolan.github.io/jq/download/) - -> Note: in linux systems, it is possible to run the docker command without prefacing -> with sudo. For more information, please refer to [the Post-installation steps -> for linux](https://docs.docker.com/install/linux/linux-postinstall/) +- Kubectl +```bash +# Install kubectl +az aks install-cli +``` +- Helm +```bash +# install helm 3 +curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash +``` Clone or download this repo locally. @@ -142,8 +147,6 @@ echo $CLUSTER_NAME Download kubectl and create a Kubernetes namespace. ```bash -# Install kubectl -az aks install-cli # Get the Kubernetes cluster credentials az aks get-credentials --resource-group=rg-shipping-dronedelivery-${LOCATION} --name=$CLUSTER_NAME @@ -152,13 +155,6 @@ az aks get-credentials --resource-group=rg-shipping-dronedelivery-${LOCATION} -- kubectl create namespace backend-dev ``` -Install and initialize Helm. - -```bash -# install helm 3 -curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash -``` - Integrate Application Insights instance. ```bash From cbc5e33cc92596e124cae8a1ea82fa34ece87cac Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Wed, 29 Jan 2025 13:57:11 +0300 Subject: [PATCH 07/44] Update deployment.md --- deployment.md | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/deployment.md b/deployment.md index 6c091d80..c8674f4c 100644 --- a/deployment.md +++ b/deployment.md @@ -7,17 +7,11 @@ - [Azure CLI 2.53.1 or newer](https://docs.microsoft.com/cli/azure/install-azure-cli) - [JQ](https://stedolan.github.io/jq/download/) - Kubectl -```bash -# Install kubectl -az aks install-cli -``` +(az aks install-cli) - Helm -```bash -# install helm 3 -curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash -``` +(curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash) -Clone or download this repo locally. +## Clone or download this repo locally. ```bash git clone --recurse-submodules https://github.com/mspnp/microservices-reference-implementation.git && \ @@ -46,7 +40,7 @@ az login az deployment sub create --name workload-stamp-prereqs --location ${LOCATION} --template-file ./workload-stamp-prereqs.bicep ``` -:book: This pre-flight Bicep template is creating a general purpose resource group as well as one dedicated for the Azure Container Registry. Additionally five User Identites are provisioned as part of this too that will be later associated to every containerized microservice. This is because they will need Azure RBAC roles over the Azure KeyVault to read secrets in runtime. The resources will be created on the resouce group location and each resource group will contain the region as part of their names +:book: This pre-flight Bicep template is creating a general purpose resource group as well as one dedicated for the Azure Container Registry. Additionally five User Identites are provisioned that will be later associated to every containerized microservice. This is because they will need Azure RBAC roles over the Azure KeyVault to read secrets in runtime. The resources will be created on the resouce group location and each resource group will contain the region as part of their names ### Get the workload user assigned identities From b5bd9b0fd064f2278b23d92bc5c48026002fee42 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Wed, 29 Jan 2025 14:06:10 +0300 Subject: [PATCH 08/44] Update deployment.md --- deployment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment.md b/deployment.md index c8674f4c..2a981e8d 100644 --- a/deployment.md +++ b/deployment.md @@ -37,7 +37,7 @@ az login ### Deploy the workload's prerequisites ```bash -az deployment sub create --name workload-stamp-prereqs --location ${LOCATION} --template-file ./workload-stamp-prereqs.bicep +az deployment sub create --name $PREREQS_DEPLOYMENT_NAME --location ${LOCATION} --template-file ./workload/workload-stamp-prereqs.bicep --parameters resourceGroupLocation=${LOCATION} ``` :book: This pre-flight Bicep template is creating a general purpose resource group as well as one dedicated for the Azure Container Registry. Additionally five User Identites are provisioned that will be later associated to every containerized microservice. This is because they will need Azure RBAC roles over the Azure KeyVault to read secrets in runtime. The resources will be created on the resouce group location and each resource group will contain the region as part of their names From f57bf3a95e00cfd6730b1bed7ada70c4c74659e5 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Wed, 29 Jan 2025 14:08:53 +0300 Subject: [PATCH 09/44] Update deployment.md --- deployment.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment.md b/deployment.md index 2a981e8d..5f4bf197 100644 --- a/deployment.md +++ b/deployment.md @@ -37,10 +37,10 @@ az login ### Deploy the workload's prerequisites ```bash -az deployment sub create --name $PREREQS_DEPLOYMENT_NAME --location ${LOCATION} --template-file ./workload/workload-stamp-prereqs.bicep --parameters resourceGroupLocation=${LOCATION} +az deployment sub create --name workload-stamp-prereqs --location ${LOCATION} --template-file ./workload/workload-stamp-prereqs.bicep --parameters resourceGroupLocation=${LOCATION} ``` -:book: This pre-flight Bicep template is creating a general purpose resource group as well as one dedicated for the Azure Container Registry. Additionally five User Identites are provisioned that will be later associated to every containerized microservice. This is because they will need Azure RBAC roles over the Azure KeyVault to read secrets in runtime. The resources will be created on the resouce group location and each resource group will contain the region as part of their names +:book: This pre-flight Bicep template creates two resource groups as well as one dedicated for the Azure Container Registry. Additionally five User Identites are provisioned that will be later associated to every containerized microservice. This is because they will need Azure RBAC roles over the Azure KeyVault to read secrets in runtime. The resources will be created on the resouce group location and each resource group will contain the region as part of their names ### Get the workload user assigned identities From 2247f759026ba6ec4b6aca62812ec0ffab8587ae Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Wed, 29 Jan 2025 14:14:50 +0300 Subject: [PATCH 10/44] Update deployment.md --- deployment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment.md b/deployment.md index 5f4bf197..46699a9f 100644 --- a/deployment.md +++ b/deployment.md @@ -40,7 +40,7 @@ az login az deployment sub create --name workload-stamp-prereqs --location ${LOCATION} --template-file ./workload/workload-stamp-prereqs.bicep --parameters resourceGroupLocation=${LOCATION} ``` -:book: This pre-flight Bicep template creates two resource groups as well as one dedicated for the Azure Container Registry. Additionally five User Identites are provisioned that will be later associated to every containerized microservice. This is because they will need Azure RBAC roles over the Azure KeyVault to read secrets in runtime. The resources will be created on the resouce group location and each resource group will contain the region as part of their names +:book: This pre-flight Bicep template creates two resource groups. Additionally five User Identites are provisioned that will be later associated to every containerized microservice. This is because they will need Azure RBAC roles over the Azure KeyVault to read secrets in runtime. The resources will be created on the resouce group location and each resource group will contain the region as part of their names ### Get the workload user assigned identities From 06e7b22356f40dd9a666eb9b125fa3ada1e5b8dd Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Wed, 29 Jan 2025 14:39:14 +0300 Subject: [PATCH 11/44] Update deployment.md --- deployment.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/deployment.md b/deployment.md index 46699a9f..ebc359b5 100644 --- a/deployment.md +++ b/deployment.md @@ -52,14 +52,10 @@ PACKAGE_ID_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATI INGESTION_ID_PRINCIPAL_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-ingestion --query principalId -o tsv) ``` -### Deploy the workload +### Deploy the workload related resources ```bash -az deployment group create -f ./workload-stamp.bicep -g rg-shipping-dronedelivery-${LOCATION} -p droneSchedulerPrincipalId=$DRONESCHEDULER_PRINCIPAL_ID \ --p workflowPrincipalId=$WORKFLOW_PRINCIPAL_ID \ --p deliveryPrincipalId=$DELIVERY_PRINCIPAL_ID \ --p ingestionPrincipalId=$INGESTION_ID_PRINCIPAL_ID \ --p packagePrincipalId=$PACKAGE_ID_PRINCIPAL_ID +az deployment group create -f ./workload/workload-stamp.bicep -g rg-shipping-dronedelivery-${LOCATION} -p droneSchedulerPrincipalId=$DRONESCHEDULER_PRINCIPAL_ID -p workflowPrincipalId=$WORKFLOW_PRINCIPAL_ID -p deliveryPrincipalId=$DELIVERY_PRINCIPAL_ID -p ingestionPrincipalId=$INGESTION_ID_PRINCIPAL_ID -p packagePrincipalId=$PACKAGE_ID_PRINCIPAL_ID ``` ### Assign ACR variables From 41b126bb9eb957c670315a2506eae4a53c4c8287 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Wed, 29 Jan 2025 15:31:39 +0300 Subject: [PATCH 12/44] Update deployment.md --- deployment.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/deployment.md b/deployment.md index ebc359b5..893039a7 100644 --- a/deployment.md +++ b/deployment.md @@ -72,31 +72,31 @@ ACR_SERVER=$(az acr show -n $ACR_NAME --query loginServer -o tsv) 1. Build the Delivery service. ```bash -az acr build -r $ACR_NAME -t $ACR_SERVER/delivery:0.1.0 ./src/shipping/delivery/. +az acr build -r $ACR_NAME -t $ACR_SERVER/delivery:0.1.0 ./workload/src/shipping/delivery/. ``` 2. Build the Ingestion service. ```bash -az acr build -r $ACR_NAME -t $ACR_SERVER/ingestion:0.1.0 ./src/shipping/ingestion/. +az acr build -r $ACR_NAME -t $ACR_SERVER/ingestion:0.1.0 ./workload/src/shipping/ingestion/. ``` 3. Build the Workflow service. ```bash -az acr build -r $ACR_NAME -t $ACR_SERVER/workflow:0.1.0 ./src/shipping/workflow/. +az acr build -r $ACR_NAME -t $ACR_SERVER/workflow:0.1.0 ./workload/src/shipping/workflow/. ``` 4. Build the DroneScheduler service. ```bash -az acr build -r $ACR_NAME -f ./src/shipping/dronescheduler/Dockerfile -t $ACR_SERVER/dronescheduler:0.1.0 ./src/shipping/. +az acr build -r $ACR_NAME -f ./workload/src/shipping/dronescheduler/Dockerfile -t $ACR_SERVER/dronescheduler:0.1.0 ./workload/src/shipping/. ``` 5. Build the Package service. ```bash -az acr build -r $ACR_NAME -t $ACR_SERVER/package:0.1.0 ./src/shipping/package/. +az acr build -r $ACR_NAME -t $ACR_SERVER/package:0.1.0 ./workload/src/shipping/package/. ``` ## Deploy the managed cluster and all related resources (This step takes about 15 minutes) From c1f92f33bf038cc7cc291ac308e236d8c4d37d7e Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Wed, 29 Jan 2025 16:27:47 +0300 Subject: [PATCH 13/44] Update deployment.md --- deployment.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/deployment.md b/deployment.md index 893039a7..c88b9d58 100644 --- a/deployment.md +++ b/deployment.md @@ -111,13 +111,10 @@ export TENANT_ID=$(az account show --query tenantId --output tsv) export DEPLOYMENT_SUFFIX=$(date +%S%N) -export KUBERNETES_VERSION=$(az aks get-versions -l $LOCATION --query "values[?isDefault].version" -o tsv) - export DEPLOYMENT_NAME=azuredeploy-$DEPLOYMENT_SUFFIX az deployment group create -g rg-shipping-dronedelivery-${LOCATION} --name $DEPLOYMENT_NAME --template-file azuredeploy.bicep \ --parameters servicePrincipalClientId=$SP_APP_ID \ servicePrincipalClientSecret=$SP_CLIENT_SECRET \ - kubernetesVersion=$KUBERNETES_VERSION \ deliveryIdName=uid-delivery \ ingestionIdName=uid-ingestion \ packageIdName=uid-package \ @@ -134,7 +131,7 @@ export CLUSTER_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LO echo $CLUSTER_NAME ``` -Download kubectl and create a Kubernetes namespace. +Get the AKS cluster credentials and create a Kubernetes namespace. ```bash @@ -148,9 +145,6 @@ kubectl create namespace backend-dev Integrate Application Insights instance. ```bash -# Acquire Instrumentation Key -export AI_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.appInsightsName.value -o tsv) -echo $AI_NAME # add RBAC for AppInsights kubectl apply -f k8s/k8s-rbac-ai.yaml From e7e26d3311669cccafc86770747dfac1acd40d39 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Wed, 29 Jan 2025 16:30:29 +0300 Subject: [PATCH 14/44] Update azuredeploy.bicep --- azuredeploy.bicep | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azuredeploy.bicep b/azuredeploy.bicep index 05a833c4..d54b47ec 100644 --- a/azuredeploy.bicep +++ b/azuredeploy.bicep @@ -40,8 +40,8 @@ param osType string = 'Linux' @maxValue(1023) param osDiskSizeGB int = 0 -@description('The version of Kubernetes. It must be supported in the target location.') -param kubernetesVersion string +//@description('The version of Kubernetes. It must be supported in the target location.') +//param kubernetesVersion string @description('Type of the storage account that will store Redis Cache.') @allowed([ @@ -101,7 +101,7 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-09-02-previ environment: 'shared cluster' } properties: { - kubernetesVersion: kubernetesVersion +// kubernetesVersion: kubernetesVersion nodeResourceGroup: nodeResourceGroupName dnsPrefix: aksClusterName agentPoolProfiles: [ From 182b9cad48ac1b65972cc994a5f0ba722ee60e48 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Wed, 29 Jan 2025 20:05:58 +0300 Subject: [PATCH 15/44] Update deployment.md --- deployment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment.md b/deployment.md index c88b9d58..c8575f97 100644 --- a/deployment.md +++ b/deployment.md @@ -120,7 +120,7 @@ az deployment group create -g rg-shipping-dronedelivery-${LOCATION} --name $DEPL packageIdName=uid-package \ droneSchedulerIdName=uid-dronescheduler \ workflowIdName=uid-workflow \ - acrResourceGroupName=rg-shipping-dronedelivery-${LOCATION}-acr \ + acrResourceGroupName=rg-shipping-dronedelivery-${LOCATION} \ acrName=$ACR_NAME ``` From 05fcec7a9fe41d42f217b94463e2c31fa6526c4f Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Wed, 29 Jan 2025 20:14:40 +0300 Subject: [PATCH 16/44] Update deployment.md --- deployment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment.md b/deployment.md index c8575f97..c88b9d58 100644 --- a/deployment.md +++ b/deployment.md @@ -120,7 +120,7 @@ az deployment group create -g rg-shipping-dronedelivery-${LOCATION} --name $DEPL packageIdName=uid-package \ droneSchedulerIdName=uid-dronescheduler \ workflowIdName=uid-workflow \ - acrResourceGroupName=rg-shipping-dronedelivery-${LOCATION} \ + acrResourceGroupName=rg-shipping-dronedelivery-${LOCATION}-acr \ acrName=$ACR_NAME ``` From 16af31b52ee4be4cacb67656c100eac31e80b6b8 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Wed, 29 Jan 2025 20:18:37 +0300 Subject: [PATCH 17/44] Update deployment.md --- deployment.md | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/deployment.md b/deployment.md index c88b9d58..9c488f4f 100644 --- a/deployment.md +++ b/deployment.md @@ -124,14 +124,13 @@ az deployment group create -g rg-shipping-dronedelivery-${LOCATION} --name $DEPL acrName=$ACR_NAME ``` -Get the cluster name output from Azure Deploy. +## Get the cluster name output from Azure Deploy. ```bash export CLUSTER_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n $DEPLOYMENT_NAME --query properties.outputs.aksClusterName.value -o tsv) -echo $CLUSTER_NAME ``` -Get the AKS cluster credentials and create a Kubernetes namespace. +## Get the AKS cluster credentials and create a Kubernetes namespace. ```bash @@ -142,7 +141,7 @@ az aks get-credentials --resource-group=rg-shipping-dronedelivery-${LOCATION} -- kubectl create namespace backend-dev ``` -Integrate Application Insights instance. +## Add cluster role and role binding for application insights. ```bash @@ -150,22 +149,6 @@ Integrate Application Insights instance. kubectl apply -f k8s/k8s-rbac-ai.yaml ``` -## Verify that the secrets-store pods are running in the kube-system namespace - -```bash -kubectl get pods -n kube-system -``` - -You should see an output similar to this: - -```bash -NAME READY STATUS RESTARTS AGE -aks-secrets-store-csi-driver-4bjzx 3/3 Running 2 28m -aks-secrets-store-csi-driver-b22bj 3/3 Running 1 28m -aks-secrets-store-provider-azure-2k5mx 1/1 Running 0 28m -aks-secrets-store-provider-azure-l5w98 1/1 Running 0 28m -``` - ## Collect details of managed ingress controller. From 5d881120a848ccdc2b702d9069c863a3d3cb0656 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Wed, 29 Jan 2025 20:29:54 +0300 Subject: [PATCH 18/44] Update deployment.md --- deployment.md | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/deployment.md b/deployment.md index 9c488f4f..aa9389a5 100644 --- a/deployment.md +++ b/deployment.md @@ -20,8 +20,6 @@ cd microservices-reference-implementation/ The deployment steps shown here use Bash shell commands. On Windows, you can use the [Windows Subsystem for Linux](https://docs.microsoft.com/windows/wsl/about) to run Bash. -## Deploy an Azure Container Registry (ACR) - Set environment variables. ```bash @@ -99,7 +97,7 @@ az acr build -r $ACR_NAME -f ./workload/src/shipping/dronescheduler/Dockerfile - az acr build -r $ACR_NAME -t $ACR_SERVER/package:0.1.0 ./workload/src/shipping/package/. ``` -## Deploy the managed cluster and all related resources (This step takes about 15 minutes) +## Deploy the managed cluster and all related resources ```bash export RESOURCE_GROUP_ID=$(az group show --name rg-shipping-dronedelivery-${LOCATION} --query id --output tsv) @@ -124,13 +122,13 @@ az deployment group create -g rg-shipping-dronedelivery-${LOCATION} --name $DEPL acrName=$ACR_NAME ``` -## Get the cluster name output from Azure Deploy. +### Get the cluster name output from Azure Deploy. ```bash export CLUSTER_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n $DEPLOYMENT_NAME --query properties.outputs.aksClusterName.value -o tsv) ``` -## Get the AKS cluster credentials and create a Kubernetes namespace. +### Get the AKS cluster credentials and create a Kubernetes namespace. ```bash @@ -141,7 +139,7 @@ az aks get-credentials --resource-group=rg-shipping-dronedelivery-${LOCATION} -- kubectl create namespace backend-dev ``` -## Add cluster role and role binding for application insights. +### Configure RBAC permissions for Azure application insights. ```bash @@ -149,7 +147,7 @@ kubectl create namespace backend-dev kubectl apply -f k8s/k8s-rbac-ai.yaml ``` -## Collect details of managed ingress controller. +### Collect details of managed ingress controller. ```bash @@ -163,7 +161,7 @@ export EXTERNAL_INGEST_FQDN=$(az network public-ip update --ids $INGRESS_LOAD_BA ``` -## Create self-signed certificate for TLS +### Create self-signed certificate for TLS > :warning: WARNING > @@ -182,13 +180,13 @@ openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ -subj "/CN=${EXTERNAL_INGEST_FQDN}/O=fabrikam" ``` -## Setup cluster resource quota +### Setup cluster resource quota ```bash kubectl apply -f k8s/k8s-resource-quotas-dev.yaml ``` -## Get the OIDC Issuer URL +### Get the OIDC Issuer URL ```bash export AKS_OIDC_ISSUER="$(az aks show -n $CLUSTER_NAME -g rg-shipping-dronedelivery-${LOCATION} --query "oidcIssuerProfile.issuerUrl" -otsv)" From c851e4891e194e3ad0c71923b4626fb871a225c3 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Wed, 29 Jan 2025 20:37:26 +0300 Subject: [PATCH 19/44] Update deployment.md --- deployment.md | 37 ++++++++++++++----------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/deployment.md b/deployment.md index aa9389a5..0b411ab8 100644 --- a/deployment.md +++ b/deployment.md @@ -147,8 +147,19 @@ kubectl create namespace backend-dev kubectl apply -f k8s/k8s-rbac-ai.yaml ``` -### Collect details of managed ingress controller. +### Set resource quotas for the namespace. + +```bash +kubectl apply -f k8s/k8s-resource-quotas-dev.yaml +``` + +### Get the OIDC Issuer URL + +```bash +export AKS_OIDC_ISSUER="$(az aks show -n $CLUSTER_NAME -g rg-shipping-dronedelivery-${LOCATION} --query "oidcIssuerProfile.issuerUrl" -otsv)" +``` +### Collect details of managed ingress controller. ```bash @@ -170,7 +181,6 @@ export EXTERNAL_INGEST_FQDN=$(az network public-ip update --ids $INGRESS_LOAD_BA > For your production cluster, use your > security best practices for digital certificates creation and lifetime management. - ```bash # Create a self-signed certificate for TLS @@ -180,18 +190,6 @@ openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ -subj "/CN=${EXTERNAL_INGEST_FQDN}/O=fabrikam" ``` -### Setup cluster resource quota - -```bash -kubectl apply -f k8s/k8s-resource-quotas-dev.yaml -``` - -### Get the OIDC Issuer URL - -```bash -export AKS_OIDC_ISSUER="$(az aks show -n $CLUSTER_NAME -g rg-shipping-dronedelivery-${LOCATION} --query "oidcIssuerProfile.issuerUrl" -otsv)" -``` - ## Deploy the Delivery service Extract resource details from deployment. @@ -203,22 +201,15 @@ export COLLECTION_NAME="${DATABASE_NAME}-col" && \ export DELIVERY_KEYVAULT_URI=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.deliveryKeyVaultUri.value -o tsv) && \ export DELIVERY_KEYVAULT_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.deliveryKeyVaultName.value -o tsv) && \ export DELIVERY_PRINCIPAL_CLIENT_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-delivery --query clientId -o tsv) -``` - -Deploy the Delivery service. -```bash # Create secrets -# Note: Ingress TLS key and certificate secrets cannot be exported as outputs in ARM deployments # The current user is given permission to import secrets and then it is deleted right after the secret creation command is executed -export SIGNED_IN_OBJECT_ID=$(az ad signed-in-user show --query 'id' -o tsv) +export SIGNED_IN_OBJECT_ID=$(az ad signed-in-user show --query 'id' -o tsv) export DELIVERY_KEYVAULT_ID=$(az resource show -g rg-shipping-dronedelivery-${LOCATION} -n $DELIVERY_KEYVAULT_NAME --resource-type 'Microsoft.KeyVault/vaults' --query id --output tsv) az role assignment create --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $DELIVERY_KEYVAULT_ID - az keyvault secret set --name Delivery-Ingress-Tls-Key --vault-name $DELIVERY_KEYVAULT_NAME --value "$(cat ingestion-ingress-tls.key)" az keyvault secret set --name Delivery-Ingress-Tls-Crt --vault-name $DELIVERY_KEYVAULT_NAME --value "$(cat ingestion-ingress-tls.crt)" - az role assignment delete --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $DELIVERY_KEYVAULT_ID #Setup your managed identity to trust your Kubernetes service account @@ -248,7 +239,7 @@ helm install delivery-v0.1.0-dev delivery-v0.1.0.tgz \ --namespace backend-dev \ --dependency-update -# Verify the pod is created +# Verify the helm deployment status. helm status delivery-v0.1.0-dev --namespace backend-dev ``` From a0c94b34264df36cb65e90408e62b4065f95b222 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Wed, 29 Jan 2025 20:45:22 +0300 Subject: [PATCH 20/44] Update deployment.md --- deployment.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deployment.md b/deployment.md index 0b411ab8..824455b3 100644 --- a/deployment.md +++ b/deployment.md @@ -208,6 +208,11 @@ export DELIVERY_PRINCIPAL_CLIENT_ID=$(az identity show -g rg-shipping-dronedeliv export SIGNED_IN_OBJECT_ID=$(az ad signed-in-user show --query 'id' -o tsv) export DELIVERY_KEYVAULT_ID=$(az resource show -g rg-shipping-dronedelivery-${LOCATION} -n $DELIVERY_KEYVAULT_NAME --resource-type 'Microsoft.KeyVault/vaults' --query id --output tsv) az role assignment create --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $DELIVERY_KEYVAULT_ID + +#wait for role assignment to finish. +sleep 60 +until az ad sp show --id $SIGNED_IN_OBJECT_ID &> /dev/null ; do echo "Waiting for Microsoft Entra ID propagation" && sleep 5; done + az keyvault secret set --name Delivery-Ingress-Tls-Key --vault-name $DELIVERY_KEYVAULT_NAME --value "$(cat ingestion-ingress-tls.key)" az keyvault secret set --name Delivery-Ingress-Tls-Crt --vault-name $DELIVERY_KEYVAULT_NAME --value "$(cat ingestion-ingress-tls.crt)" az role assignment delete --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $DELIVERY_KEYVAULT_ID From 8c80cca2553a9969d43ff7b6d91e67047fdde3a1 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Wed, 29 Jan 2025 22:00:29 +0300 Subject: [PATCH 21/44] Update deployment.md --- deployment.md | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/deployment.md b/deployment.md index 824455b3..0f9e4f02 100644 --- a/deployment.md +++ b/deployment.md @@ -253,27 +253,13 @@ helm status delivery-v0.1.0-dev --namespace backend-dev Extract resource details from deployment. ```bash -export COSMOSDB_NAME_PACKAGE=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.packageMongoDbName.value -o tsv) export PACKAGE_KEYVAULT_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.packageKeyVaultName.value -o tsv) export PACKAGE_ID_CLIENT_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-package --query clientId -o tsv) -``` -Deploy the Package service. +# Deploy the Package service. -```bash -# Create secret -# Note: Connection strings cannot be exported as outputs in ARM deployments -# The current user is given permission to import secrets and then it is deleted right after the secret creation command is executed -export COSMOSDB_CONNECTION_PACKAGE=$(az cosmosdb keys list --type connection-strings --name $COSMOSDB_NAME_PACKAGE --resource-group rg-shipping-dronedelivery-${LOCATION} --query "connectionStrings[0].connectionString" -o tsv | sed 's/==/%3D%3D/g') && \ export COSMOSDB_COL_NAME_PACKAGE=packages -export PACKAGE_KEYVAULT_ID=$(az resource show -g rg-shipping-dronedelivery-${LOCATION} -n $PACKAGE_KEYVAULT_NAME --resource-type 'Microsoft.KeyVault/vaults' --query id --output tsv) -az role assignment create --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $PACKAGE_KEYVAULT_ID - -az keyvault secret set --name CosmosDb--ConnectionString --vault-name $PACKAGE_KEYVAULT_NAME --value $COSMOSDB_CONNECTION_PACKAGE - -az role assignment delete --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $PACKAGE_KEYVAULT_ID - # Setup your managed identity to trust your Kubernetes service account az identity federated-credential create --name credential-for-package --identity-name uid-package --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:package-sa-v0.1.0 From 029f8f78569250897cb52d79c2beeee85859a251 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Wed, 29 Jan 2025 22:02:19 +0300 Subject: [PATCH 22/44] Update deployment.md --- deployment.md | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/deployment.md b/deployment.md index 0f9e4f02..26246a68 100644 --- a/deployment.md +++ b/deployment.md @@ -299,11 +299,6 @@ export WORKFLOW_NAMESPACE_NAME=$(az deployment group show -g rg-shipping-dronede export WORKFLOW_NAMESPACE_ENDPOINT=$(az servicebus namespace show -g rg-shipping-dronedelivery-${LOCATION} -n $WORKFLOW_NAMESPACE_NAME --query serviceBusEndpoint -o tsv) export WORKFLOW_NAMESPACE_SAS_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.workflowServiceAccessKeyName.value -o tsv) -``` - -Deploy the Workflow service. - -```bash # Setup your managed identity to trust your Kubernetes service account az identity federated-credential create --name credential-for-workflow --identity-name uid-workflow --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:workflow-sa-v0.1.0 @@ -351,11 +346,7 @@ az keyvault secret set --name Ingestion-Ingress-Tls-Key --vault-name $INGESTION_ az keyvault secret set --name Ingestion-Ingress-Tls-Crt --vault-name $INGESTION_KEYVAULT_NAME --value "$(cat ingestion-ingress-tls.crt)" az role assignment delete --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $INGESTION_KEYVAULT_ID -``` -Deploy the Ingestion service - -```bash # Setup your managed identity to trust your Kubernetes service account az identity federated-credential create --name credential-for-ingestion --identity-name uid-ingestion --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:ingestion-sa-v0.1.0 @@ -399,11 +390,7 @@ export AUTH_KEY=$(az cosmosdb keys list -n $DRONESCHEDULER_COSMOSDB_NAME -g rg-s export DRONESCHEDULER_CLIENT_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-dronescheduler --query clientId -o tsv) && \ export DATABASE_NAME="invoicing" && \ export COLLECTION_NAME="utilization" -``` -Deploy the dronescheduler service. - -```bash # Setup your managed identity to trust your Kubernetes service account az identity federated-credential create --name credential-for-dronescheduler --identity-name uid-dronescheduler --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:dronescheduler-sa-v0.1.0 From 5b3812d90c3a26996fde3afad580c63459dad6a1 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Thu, 30 Jan 2025 17:00:59 +0300 Subject: [PATCH 23/44] Update azuredeploy.bicep - removing service principal. --- azuredeploy.bicep | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/azuredeploy.bicep b/azuredeploy.bicep index d54b47ec..13c804b2 100644 --- a/azuredeploy.bicep +++ b/azuredeploy.bicep @@ -94,6 +94,16 @@ resource miClusterControlPlane 'Microsoft.ManagedIdentity/userAssignedIdentities location: location } +//provide contributor role to the RG to AKS managed identity. +resource roleAssignment 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = { + name: guid(resourceGroup().id, aksServicePrincipal.id) + scope: resourceGroup() + properties: { + principalId: miClusterControlPlane.properties.principalId + roleDefinitionId: 'b24988ac-6180-42a0-ab88-20f7382dd24c' + } +} + resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-09-02-preview' = { name: aksClusterName location: location @@ -122,10 +132,6 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-09-02-previ mode: 'User' } ] - servicePrincipalProfile: { - clientId: servicePrincipalClientId - secret: servicePrincipalClientSecret - } addonProfiles: { omsagent: { config: { From 535fa687600aa73abce70475c1f782b89cae6840 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Thu, 30 Jan 2025 17:03:03 +0300 Subject: [PATCH 24/44] Update deployment.md --- deployment.md | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/deployment.md b/deployment.md index 26246a68..e500b397 100644 --- a/deployment.md +++ b/deployment.md @@ -100,20 +100,12 @@ az acr build -r $ACR_NAME -t $ACR_SERVER/package:0.1.0 ./workload/src/shipping/p ## Deploy the managed cluster and all related resources ```bash -export RESOURCE_GROUP_ID=$(az group show --name rg-shipping-dronedelivery-${LOCATION} --query id --output tsv) - -export SP_DETAILS=$(az ad sp create-for-rbac --role="Contributor" --scopes $RESOURCE_GROUP_ID -o json) && \ -export SP_APP_ID=$(echo $SP_DETAILS | jq ".appId" -r) && \ -export SP_CLIENT_SECRET=$(echo $SP_DETAILS | jq ".password" -r) -export TENANT_ID=$(az account show --query tenantId --output tsv) export DEPLOYMENT_SUFFIX=$(date +%S%N) - export DEPLOYMENT_NAME=azuredeploy-$DEPLOYMENT_SUFFIX + az deployment group create -g rg-shipping-dronedelivery-${LOCATION} --name $DEPLOYMENT_NAME --template-file azuredeploy.bicep \ ---parameters servicePrincipalClientId=$SP_APP_ID \ - servicePrincipalClientSecret=$SP_CLIENT_SECRET \ - deliveryIdName=uid-delivery \ +--parameters deliveryIdName=uid-delivery \ ingestionIdName=uid-ingestion \ packageIdName=uid-package \ droneSchedulerIdName=uid-dronescheduler \ @@ -165,7 +157,6 @@ export AKS_OIDC_ISSUER="$(az aks show -n $CLUSTER_NAME -g rg-shipping-dronedeliv # Obtain the load balancer ip address of managed ingress and assign a domain name export INGRESS_LOAD_BALANCER_IP=$(kubectl get service -n app-routing-system nginx -o jsonpath="{.status.loadBalancer.ingress[0].ip}" 2> /dev/null) - export INGRESS_LOAD_BALANCER_IP_ID=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$INGRESS_LOAD_BALANCER_IP')].[id]" --output tsv) && \ export EXTERNAL_INGEST_DNS_NAME="dronedelivery-${LOCATION}-${RANDOM}-ing" && \ export EXTERNAL_INGEST_FQDN=$(az network public-ip update --ids $INGRESS_LOAD_BALANCER_IP_ID --dns-name $EXTERNAL_INGEST_DNS_NAME --query "dnsSettings.fqdn" --output tsv) From 34080d0dfb5718c42d3a04f254c3682bfd46dd83 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Thu, 30 Jan 2025 17:05:05 +0300 Subject: [PATCH 25/44] Update azuredeploy.bicep --- azuredeploy.bicep | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/azuredeploy.bicep b/azuredeploy.bicep index 13c804b2..cf35e8dd 100644 --- a/azuredeploy.bicep +++ b/azuredeploy.bicep @@ -22,13 +22,6 @@ param ingestionIdName string @description('Name of the package managed identity') param packageIdName string -@description('Client ID (used by cloudprovider)') -param servicePrincipalClientId string - -@description('The Service Principal Client Secret.') -@secure() -param servicePrincipalClientSecret string - @description('The type of operating system.') @allowed([ 'Linux' @@ -96,7 +89,7 @@ resource miClusterControlPlane 'Microsoft.ManagedIdentity/userAssignedIdentities //provide contributor role to the RG to AKS managed identity. resource roleAssignment 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = { - name: guid(resourceGroup().id, aksServicePrincipal.id) + name: guid(resourceGroup().id, miClusterControlPlane.id) scope: resourceGroup() properties: { principalId: miClusterControlPlane.properties.principalId From f740d1d1d436235617e8b54e083e98a4c5f0fc80 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Thu, 30 Jan 2025 18:58:20 +0300 Subject: [PATCH 26/44] Update deployment.md --- deployment.md | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/deployment.md b/deployment.md index e500b397..4429bf50 100644 --- a/deployment.md +++ b/deployment.md @@ -67,31 +67,31 @@ ACR_SERVER=$(az acr show -n $ACR_NAME --query loginServer -o tsv) ### Steps -1. Build the Delivery service. +1. Build and push the Delivery service container image to ACR. ```bash az acr build -r $ACR_NAME -t $ACR_SERVER/delivery:0.1.0 ./workload/src/shipping/delivery/. ``` -2. Build the Ingestion service. +2. Build and push the Ingestion service container image to ACR. ```bash az acr build -r $ACR_NAME -t $ACR_SERVER/ingestion:0.1.0 ./workload/src/shipping/ingestion/. ``` -3. Build the Workflow service. +3. Build and push the Workflow service container image to ACR. ```bash az acr build -r $ACR_NAME -t $ACR_SERVER/workflow:0.1.0 ./workload/src/shipping/workflow/. ``` -4. Build the DroneScheduler service. +4. Build and push the DroneScheduler service container image to ACR. ```bash az acr build -r $ACR_NAME -f ./workload/src/shipping/dronescheduler/Dockerfile -t $ACR_SERVER/dronescheduler:0.1.0 ./workload/src/shipping/. ``` -5. Build the Package service. +5. Build and push the Package service container image to ACR. ```bash az acr build -r $ACR_NAME -t $ACR_SERVER/package:0.1.0 ./workload/src/shipping/package/. @@ -101,10 +101,7 @@ az acr build -r $ACR_NAME -t $ACR_SERVER/package:0.1.0 ./workload/src/shipping/p ```bash -export DEPLOYMENT_SUFFIX=$(date +%S%N) -export DEPLOYMENT_NAME=azuredeploy-$DEPLOYMENT_SUFFIX - -az deployment group create -g rg-shipping-dronedelivery-${LOCATION} --name $DEPLOYMENT_NAME --template-file azuredeploy.bicep \ +az deployment group create -g rg-shipping-dronedelivery-${LOCATION} --name managed-cluster-deployment --template-file azuredeploy.bicep \ --parameters deliveryIdName=uid-delivery \ ingestionIdName=uid-ingestion \ packageIdName=uid-package \ From a02ee1981b0648b06d0a0728b8e01e4fa97fc3b7 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Thu, 30 Jan 2025 19:47:40 +0300 Subject: [PATCH 27/44] Update azuredeploy.bicep --- azuredeploy.bicep | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/azuredeploy.bicep b/azuredeploy.bicep index cf35e8dd..e05854db 100644 --- a/azuredeploy.bicep +++ b/azuredeploy.bicep @@ -42,21 +42,22 @@ param osDiskSizeGB int = 0 'Standard_ZRS' 'Standard_GRS' ]) + param deliveryRedisStorageType string = 'Standard_LRS' +param workspaceName string + var clusterNamePrefix = 'aks' var managedIdentityOperatorRoleId = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'f1a07417-d97a-45cb-824c-7a7467783830') -var deliveryRedisStorageName = 'rsto${uniqueString(resourceGroup().id)}' +var deliveryRedisStorageName = 'stdelivery${uniqueString(resourceGroup().id)}' var nestedACRDeploymentName = 'azuredeploy-acr-${acrResourceGroupName}' var aksLogAnalyticsNamePrefix = 'logsAnalytics' var monitoringMetricsPublisherRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '3913510d-42f4-4e42-8a64-420c390055eb') var nodeResourceGroupName = 'rg-${aksClusterName}-nodepools' -var aksClusterName = uniqueString(clusterNamePrefix, resourceGroup().id) +var aksClusterName = aks-${uniqueString(clusterNamePrefix, resourceGroup().id)} var agentCount = 2 var agentVMSize = 'Standard_D2_v2' -var workspaceName = 'la-${uniqueString(aksLogAnalyticsNamePrefix, resourceGroup().id)}' -var workspaceSku = 'pergb2018' -var workspaceRetentionInDays = 0 + module nestedACRDeployment './azuredeploy_nested_nestedACRDeployment.bicep' = { name: nestedACRDeploymentName @@ -67,18 +68,9 @@ module nestedACRDeployment './azuredeploy_nested_nestedACRDeployment.bicep' = { } } -resource workspace 'Microsoft.OperationalInsights/workspaces@2022-10-01' = { +//use the log analytics workspace that is already created. +resource workspace 'Microsoft.OperationalInsights/workspaces@2022-10-01' existing = { name: workspaceName - location: location - properties: { - retentionInDays: workspaceRetentionInDays - sku: { - name: workspaceSku - } - features: { - searchVersion: 1 - } - } } // The control plane identity used by the cluster. Used for networking access (VNET joining and DNS updating) @@ -173,7 +165,6 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-09-02-previ '${miClusterControlPlane.id}': {} } } - } resource deliveryRedisStorage 'Microsoft.Storage/storageAccounts@2022-09-01' = { From 2ba9c14dcf6d857ed616cd2578fe4dce3e5173ed Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Thu, 30 Jan 2025 19:58:26 +0300 Subject: [PATCH 28/44] Update azuredeploy.bicep - removing storage --- azuredeploy.bicep | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/azuredeploy.bicep b/azuredeploy.bicep index e05854db..0ed67280 100644 --- a/azuredeploy.bicep +++ b/azuredeploy.bicep @@ -36,22 +36,24 @@ param osDiskSizeGB int = 0 //@description('The version of Kubernetes. It must be supported in the target location.') //param kubernetesVersion string +/* @description('Type of the storage account that will store Redis Cache.') @allowed([ 'Standard_LRS' 'Standard_ZRS' 'Standard_GRS' ]) +*/ -param deliveryRedisStorageType string = 'Standard_LRS' +//param deliveryRedisStorageType string = 'Standard_LRS' param workspaceName string var clusterNamePrefix = 'aks' var managedIdentityOperatorRoleId = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'f1a07417-d97a-45cb-824c-7a7467783830') -var deliveryRedisStorageName = 'stdelivery${uniqueString(resourceGroup().id)}' +//var deliveryRedisStorageName = 'stdelivery${uniqueString(resourceGroup().id)}' var nestedACRDeploymentName = 'azuredeploy-acr-${acrResourceGroupName}' -var aksLogAnalyticsNamePrefix = 'logsAnalytics' +//var aksLogAnalyticsNamePrefix = 'logsAnalytics' var monitoringMetricsPublisherRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '3913510d-42f4-4e42-8a64-420c390055eb') var nodeResourceGroupName = 'rg-${aksClusterName}-nodepools' var aksClusterName = aks-${uniqueString(clusterNamePrefix, resourceGroup().id)} @@ -167,6 +169,7 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-09-02-previ } } +/* resource deliveryRedisStorage 'Microsoft.Storage/storageAccounts@2022-09-01' = { name: deliveryRedisStorageName sku: { @@ -179,6 +182,7 @@ resource deliveryRedisStorage 'Microsoft.Storage/storageAccounts@2022-09-01' = { app: 'fabrikam-delivery' } } +*/ resource clusterIdentityPublisherRoleAssigment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { name: guid(concat(resourceGroup().id), monitoringMetricsPublisherRole) From d513ac9ed3b7c424978632211363afcbc04699b1 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Thu, 30 Jan 2025 20:03:51 +0300 Subject: [PATCH 29/44] Update deployment.md --- deployment.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deployment.md b/deployment.md index 4429bf50..6f2bce61 100644 --- a/deployment.md +++ b/deployment.md @@ -101,6 +101,8 @@ az acr build -r $ACR_NAME -t $ACR_SERVER/package:0.1.0 ./workload/src/shipping/p ```bash +export LOG_ANALYTICS_WORKSPACE_ID=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.laWorkspace.value -o tsv) + az deployment group create -g rg-shipping-dronedelivery-${LOCATION} --name managed-cluster-deployment --template-file azuredeploy.bicep \ --parameters deliveryIdName=uid-delivery \ ingestionIdName=uid-ingestion \ @@ -108,7 +110,8 @@ az deployment group create -g rg-shipping-dronedelivery-${LOCATION} --name manag droneSchedulerIdName=uid-dronescheduler \ workflowIdName=uid-workflow \ acrResourceGroupName=rg-shipping-dronedelivery-${LOCATION}-acr \ - acrName=$ACR_NAME + acrName=$ACR_NAME \ + logAnalyticsWorkspaceID=$LOG_ANALYTICS_WORKSPACE_ID ``` ### Get the cluster name output from Azure Deploy. From b09139aba030269ecc65d257a0d1f4662838a9f2 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Thu, 30 Jan 2025 20:12:39 +0300 Subject: [PATCH 30/44] Update azuredeploy.bicep --- azuredeploy.bicep | 39 ++------------------------------------- 1 file changed, 2 insertions(+), 37 deletions(-) diff --git a/azuredeploy.bicep b/azuredeploy.bicep index 0ed67280..fa8f4144 100644 --- a/azuredeploy.bicep +++ b/azuredeploy.bicep @@ -36,31 +36,16 @@ param osDiskSizeGB int = 0 //@description('The version of Kubernetes. It must be supported in the target location.') //param kubernetesVersion string -/* -@description('Type of the storage account that will store Redis Cache.') -@allowed([ - 'Standard_LRS' - 'Standard_ZRS' - 'Standard_GRS' -]) -*/ - -//param deliveryRedisStorageType string = 'Standard_LRS' - -param workspaceName string +param logAnalyticsWorkspaceID string -var clusterNamePrefix = 'aks' var managedIdentityOperatorRoleId = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'f1a07417-d97a-45cb-824c-7a7467783830') -//var deliveryRedisStorageName = 'stdelivery${uniqueString(resourceGroup().id)}' var nestedACRDeploymentName = 'azuredeploy-acr-${acrResourceGroupName}' -//var aksLogAnalyticsNamePrefix = 'logsAnalytics' var monitoringMetricsPublisherRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '3913510d-42f4-4e42-8a64-420c390055eb') var nodeResourceGroupName = 'rg-${aksClusterName}-nodepools' var aksClusterName = aks-${uniqueString(clusterNamePrefix, resourceGroup().id)} var agentCount = 2 var agentVMSize = 'Standard_D2_v2' - module nestedACRDeployment './azuredeploy_nested_nestedACRDeployment.bicep' = { name: nestedACRDeploymentName scope: resourceGroup(acrResourceGroupName) @@ -70,11 +55,6 @@ module nestedACRDeployment './azuredeploy_nested_nestedACRDeployment.bicep' = { } } -//use the log analytics workspace that is already created. -resource workspace 'Microsoft.OperationalInsights/workspaces@2022-10-01' existing = { - name: workspaceName -} - // The control plane identity used by the cluster. Used for networking access (VNET joining and DNS updating) resource miClusterControlPlane 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { name: 'mi-${aksClusterName}-controlplane' @@ -122,7 +102,7 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-09-02-previ addonProfiles: { omsagent: { config: { - logAnalyticsWorkspaceResourceID: workspace.id + logAnalyticsWorkspaceResourceID: logAnalyticsWorkspaceID } enabled: true } @@ -169,21 +149,6 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-09-02-previ } } -/* -resource deliveryRedisStorage 'Microsoft.Storage/storageAccounts@2022-09-01' = { - name: deliveryRedisStorageName - sku: { - name: deliveryRedisStorageType - } - kind: 'Storage' - location: location - tags: { - displayName: 'Storage account for inflight deliveries' - app: 'fabrikam-delivery' - } -} -*/ - resource clusterIdentityPublisherRoleAssigment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { name: guid(concat(resourceGroup().id), monitoringMetricsPublisherRole) scope: aksCluster From ddeed20870735987efa6fcd97f8254b8899f3979 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Thu, 30 Jan 2025 20:18:22 +0300 Subject: [PATCH 31/44] Update azuredeploy.bicep --- azuredeploy.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azuredeploy.bicep b/azuredeploy.bicep index fa8f4144..2da6e9cc 100644 --- a/azuredeploy.bicep +++ b/azuredeploy.bicep @@ -42,7 +42,7 @@ var managedIdentityOperatorRoleId = subscriptionResourceId('Microsoft.Authorizat var nestedACRDeploymentName = 'azuredeploy-acr-${acrResourceGroupName}' var monitoringMetricsPublisherRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '3913510d-42f4-4e42-8a64-420c390055eb') var nodeResourceGroupName = 'rg-${aksClusterName}-nodepools' -var aksClusterName = aks-${uniqueString(clusterNamePrefix, resourceGroup().id)} +var aksClusterName = 'aks-${uniqueString(clusterNamePrefix, resourceGroup().id)}' var agentCount = 2 var agentVMSize = 'Standard_D2_v2' From 3b9f60c375d55c5b3f07ba4155caa36552ce74b3 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Thu, 30 Jan 2025 20:19:32 +0300 Subject: [PATCH 32/44] Update azuredeploy.bicep --- azuredeploy.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azuredeploy.bicep b/azuredeploy.bicep index 2da6e9cc..a6f80ea0 100644 --- a/azuredeploy.bicep +++ b/azuredeploy.bicep @@ -42,7 +42,7 @@ var managedIdentityOperatorRoleId = subscriptionResourceId('Microsoft.Authorizat var nestedACRDeploymentName = 'azuredeploy-acr-${acrResourceGroupName}' var monitoringMetricsPublisherRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '3913510d-42f4-4e42-8a64-420c390055eb') var nodeResourceGroupName = 'rg-${aksClusterName}-nodepools' -var aksClusterName = 'aks-${uniqueString(clusterNamePrefix, resourceGroup().id)}' +var aksClusterName = 'aks-${uniqueString(resourceGroup().id)}' var agentCount = 2 var agentVMSize = 'Standard_D2_v2' From 5c08d4024ebde9ede9ff786bf1fd48e01814392a Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Thu, 30 Jan 2025 20:31:46 +0300 Subject: [PATCH 33/44] Update azuredeploy.bicep --- azuredeploy.bicep | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/azuredeploy.bicep b/azuredeploy.bicep index a6f80ea0..da92b4d0 100644 --- a/azuredeploy.bicep +++ b/azuredeploy.bicep @@ -41,6 +41,7 @@ param logAnalyticsWorkspaceID string var managedIdentityOperatorRoleId = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'f1a07417-d97a-45cb-824c-7a7467783830') var nestedACRDeploymentName = 'azuredeploy-acr-${acrResourceGroupName}' var monitoringMetricsPublisherRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '3913510d-42f4-4e42-8a64-420c390055eb') +var contributorRoleId = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c') var nodeResourceGroupName = 'rg-${aksClusterName}-nodepools' var aksClusterName = 'aks-${uniqueString(resourceGroup().id)}' var agentCount = 2 @@ -67,7 +68,8 @@ resource roleAssignment 'Microsoft.Authorization/roleAssignments@2020-04-01-prev scope: resourceGroup() properties: { principalId: miClusterControlPlane.properties.principalId - roleDefinitionId: 'b24988ac-6180-42a0-ab88-20f7382dd24c' + roleDefinitionId: contributorRoleId + principalType: 'ServicePrincipal' } } From 129719349b0ccc9bb76fdb07cc405fb9346a1ce7 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Thu, 30 Jan 2025 21:00:48 +0300 Subject: [PATCH 34/44] Update azuredeploy.bicep --- azuredeploy.bicep | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/azuredeploy.bicep b/azuredeploy.bicep index da92b4d0..cab83cf6 100644 --- a/azuredeploy.bicep +++ b/azuredeploy.bicep @@ -64,7 +64,7 @@ resource miClusterControlPlane 'Microsoft.ManagedIdentity/userAssignedIdentities //provide contributor role to the RG to AKS managed identity. resource roleAssignment 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = { - name: guid(resourceGroup().id, miClusterControlPlane.id) + name: 'rg-contributor-${guid(resourceGroup().id, miClusterControlPlane.id)}' scope: resourceGroup() properties: { principalId: miClusterControlPlane.properties.principalId @@ -152,7 +152,7 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-09-02-previ } resource clusterIdentityPublisherRoleAssigment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(concat(resourceGroup().id), monitoringMetricsPublisherRole) + name: 'cluster-id-publisher-${guid(concat(resourceGroup().id), monitoringMetricsPublisherRole)}' scope: aksCluster properties: { roleDefinitionId: monitoringMetricsPublisherRole @@ -166,7 +166,7 @@ resource deliveryId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31 } resource deliveryIdentityOperatorRoleAssigment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid('msi-delivery', resourceGroup().id) + name: 'delivery-id-operator-${guid('msi-delivery', resourceGroup().id)}' scope: deliveryId properties: { roleDefinitionId: managedIdentityOperatorRoleId @@ -180,7 +180,7 @@ resource workflowId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31 } resource workflowIdentityOperatorRoleAssigment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid('msi-workflow', resourceGroup().id) + name: 'workflow-id-operator-${guid('msi-workflow', resourceGroup().id)}' scope: workflowId properties: { roleDefinitionId: managedIdentityOperatorRoleId @@ -194,7 +194,7 @@ resource droneSchedulerId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023 } resource droneSchedulerIdentityOperatorRoleAssigment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid('msi-dronescheduler', resourceGroup().id) + name: 'scheduler-id-operator-${guid('msi-dronescheduler', resourceGroup().id)}' scope: droneSchedulerId properties: { roleDefinitionId: managedIdentityOperatorRoleId @@ -208,7 +208,7 @@ resource ingestionId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-3 } resource ingestionIdentityOperatorRoleAssigment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid('msi-ingestion', resourceGroup().id) + name: 'ingestion-id-operator-${guid('msi-ingestion', resourceGroup().id)}' scope: ingestionId properties: { roleDefinitionId: managedIdentityOperatorRoleId @@ -222,7 +222,7 @@ resource packageId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' } resource packageIdentityOperatorRoleAssigment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid('msi-package', resourceGroup().id) + name: 'package-id-operator-${guid('msi-package', resourceGroup().id)}' scope: packageId properties: { roleDefinitionId: managedIdentityOperatorRoleId From 7027d64036a27529b0741a91f03b074c1c67445b Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Thu, 30 Jan 2025 21:05:57 +0300 Subject: [PATCH 35/44] Update azuredeploy.bicep --- azuredeploy.bicep | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/azuredeploy.bicep b/azuredeploy.bicep index cab83cf6..da92b4d0 100644 --- a/azuredeploy.bicep +++ b/azuredeploy.bicep @@ -64,7 +64,7 @@ resource miClusterControlPlane 'Microsoft.ManagedIdentity/userAssignedIdentities //provide contributor role to the RG to AKS managed identity. resource roleAssignment 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = { - name: 'rg-contributor-${guid(resourceGroup().id, miClusterControlPlane.id)}' + name: guid(resourceGroup().id, miClusterControlPlane.id) scope: resourceGroup() properties: { principalId: miClusterControlPlane.properties.principalId @@ -152,7 +152,7 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-09-02-previ } resource clusterIdentityPublisherRoleAssigment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: 'cluster-id-publisher-${guid(concat(resourceGroup().id), monitoringMetricsPublisherRole)}' + name: guid(concat(resourceGroup().id), monitoringMetricsPublisherRole) scope: aksCluster properties: { roleDefinitionId: monitoringMetricsPublisherRole @@ -166,7 +166,7 @@ resource deliveryId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31 } resource deliveryIdentityOperatorRoleAssigment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: 'delivery-id-operator-${guid('msi-delivery', resourceGroup().id)}' + name: guid('msi-delivery', resourceGroup().id) scope: deliveryId properties: { roleDefinitionId: managedIdentityOperatorRoleId @@ -180,7 +180,7 @@ resource workflowId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31 } resource workflowIdentityOperatorRoleAssigment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: 'workflow-id-operator-${guid('msi-workflow', resourceGroup().id)}' + name: guid('msi-workflow', resourceGroup().id) scope: workflowId properties: { roleDefinitionId: managedIdentityOperatorRoleId @@ -194,7 +194,7 @@ resource droneSchedulerId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023 } resource droneSchedulerIdentityOperatorRoleAssigment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: 'scheduler-id-operator-${guid('msi-dronescheduler', resourceGroup().id)}' + name: guid('msi-dronescheduler', resourceGroup().id) scope: droneSchedulerId properties: { roleDefinitionId: managedIdentityOperatorRoleId @@ -208,7 +208,7 @@ resource ingestionId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-3 } resource ingestionIdentityOperatorRoleAssigment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: 'ingestion-id-operator-${guid('msi-ingestion', resourceGroup().id)}' + name: guid('msi-ingestion', resourceGroup().id) scope: ingestionId properties: { roleDefinitionId: managedIdentityOperatorRoleId @@ -222,7 +222,7 @@ resource packageId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' } resource packageIdentityOperatorRoleAssigment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: 'package-id-operator-${guid('msi-package', resourceGroup().id)}' + name: guid('msi-package', resourceGroup().id) scope: packageId properties: { roleDefinitionId: managedIdentityOperatorRoleId From def2364bfbb9add781f25725f7a81313d78d7592 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Thu, 30 Jan 2025 21:13:29 +0300 Subject: [PATCH 36/44] Update deployment.md --- deployment.md | 1 + 1 file changed, 1 insertion(+) diff --git a/deployment.md b/deployment.md index 6f2bce61..d9364b55 100644 --- a/deployment.md +++ b/deployment.md @@ -186,6 +186,7 @@ openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ Extract resource details from deployment. ```bash +export TENANT_ID=$(az account show --query tenantId --output tsv) export COSMOSDB_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.deliveryCosmosDbName.value -o tsv) && \ export DATABASE_NAME="${COSMOSDB_NAME}-db" && \ export COLLECTION_NAME="${DATABASE_NAME}-col" && \ From 6f7936ef7e790705c44264870211150be928fc14 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Thu, 30 Jan 2025 21:17:34 +0300 Subject: [PATCH 37/44] Update deployment.md --- deployment.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deployment.md b/deployment.md index d9364b55..0cef4347 100644 --- a/deployment.md +++ b/deployment.md @@ -203,8 +203,6 @@ az role assignment create --role 'Key Vault Secrets Officer' --assignee $SIGNED_ #wait for role assignment to finish. sleep 60 -until az ad sp show --id $SIGNED_IN_OBJECT_ID &> /dev/null ; do echo "Waiting for Microsoft Entra ID propagation" && sleep 5; done - az keyvault secret set --name Delivery-Ingress-Tls-Key --vault-name $DELIVERY_KEYVAULT_NAME --value "$(cat ingestion-ingress-tls.key)" az keyvault secret set --name Delivery-Ingress-Tls-Crt --vault-name $DELIVERY_KEYVAULT_NAME --value "$(cat ingestion-ingress-tls.crt)" az role assignment delete --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $DELIVERY_KEYVAULT_ID @@ -334,6 +332,7 @@ export INGESTION_ID_CLIENT_ID=$(az identity show -g rg-shipping-dronedelivery-${ export INGESTION_KEYVAULT_ID=$(az resource show -g rg-shipping-dronedelivery-${LOCATION} -n $INGESTION_KEYVAULT_NAME --resource-type 'Microsoft.KeyVault/vaults' --query id --output tsv) az role assignment create --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $INGESTION_KEYVAULT_ID +sleep 60 az keyvault secret set --name Ingestion-Ingress-Tls-Key --vault-name $INGESTION_KEYVAULT_NAME --value "$(cat ingestion-ingress-tls.key)" az keyvault secret set --name Ingestion-Ingress-Tls-Crt --vault-name $INGESTION_KEYVAULT_NAME --value "$(cat ingestion-ingress-tls.crt)" From 2d136a77bc02576cd7a23b0cd3f9d20860ac58da Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Thu, 30 Jan 2025 22:34:36 +0300 Subject: [PATCH 38/44] Update deployment.md --- deployment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment.md b/deployment.md index 0cef4347..4ab2347e 100644 --- a/deployment.md +++ b/deployment.md @@ -117,7 +117,7 @@ az deployment group create -g rg-shipping-dronedelivery-${LOCATION} --name manag ### Get the cluster name output from Azure Deploy. ```bash -export CLUSTER_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n $DEPLOYMENT_NAME --query properties.outputs.aksClusterName.value -o tsv) +export CLUSTER_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n managed-cluster-deployment --query properties.outputs.aksClusterName.value -o tsv) ``` ### Get the AKS cluster credentials and create a Kubernetes namespace. From b98c6529683c14995ad90bc2d40ccf39b9568f5b Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Fri, 31 Jan 2025 00:07:36 +0300 Subject: [PATCH 39/44] Update deployment.md --- deployment.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deployment.md b/deployment.md index 4ab2347e..5c487c71 100644 --- a/deployment.md +++ b/deployment.md @@ -225,7 +225,7 @@ helm install delivery-v0.1.0-dev delivery-v0.1.0.tgz \ --set identity.tenantId=$TENANT_ID \ --set keyVaultName=$DELIVERY_KEYVAULT_NAME \ --set ingress.tls=true \ - --set ingress.class=nginx \ + --set ingress.class=webapprouting.kubernetes.azure.com \ --set cosmosdb.id=$DATABASE_NAME \ --set cosmosdb.collectionid=$COLLECTION_NAME \ --set keyvault.uri=$DELIVERY_KEYVAULT_URI \ @@ -265,7 +265,7 @@ helm install package-v0.1.0-dev package-v0.1.0.tgz \ --set ingress.hosts[0].name=$EXTERNAL_INGEST_FQDN \ --set ingress.hosts[0].serviceName=package \ --set ingress.hosts[0].tls=false \ - --set ingress.class=nginx \ + --set ingress.class=webapprouting.kubernetes.azure.com \ --set cosmosDb.collectionName=$COSMOSDB_COL_NAME_PACKAGE \ --set dockerregistry=$ACR_SERVER \ --set reason="Initial deployment" \ @@ -356,7 +356,7 @@ helm install ingestion-v0.1.0-dev ingestion-v0.1.0.tgz \ --set ingress.hosts[0].tls=true \ --set ingress.hosts[0].tlsSecretName=ingestion-ingress-tls \ --set ingress.tls=true \ - --set ingress.class=nginx \ + --set ingress.class=webapprouting.kubernetes.azure.com \ --set secrets.queue.keyname=IngestionServiceAccessKey \ --set secrets.queue.name=${INGESTION_QUEUE_NAME} \ --set secrets.queue.namespace=${INGESTION_QUEUE_NAMESPACE} \ @@ -394,7 +394,7 @@ helm install dronescheduler-v0.1.0-dev dronescheduler-v0.1.0.tgz \ --set ingress.hosts[0].name=$EXTERNAL_INGEST_FQDN \ --set ingress.hosts[0].serviceName=dronescheduler \ --set ingress.hosts[0].tls=false \ - --set ingress.class=nginx \ + --set ingress.class=webapprouting.kubernetes.azure.com \ --set identity.clientid=$DRONESCHEDULER_CLIENT_ID \ --set identity.serviceAccountName=dronescheduler-sa-v0.1.0 \ --set keyvault.uri=$DRONESCHEDULER_KEYVAULT_URI \ From 5f9d5ce317ff6e41f1d016c000eba9d3652581d0 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Fri, 31 Jan 2025 15:20:50 +0300 Subject: [PATCH 40/44] Update deployment.md --- deployment.md | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/deployment.md b/deployment.md index 5c487c71..20772671 100644 --- a/deployment.md +++ b/deployment.md @@ -145,10 +145,30 @@ kubectl apply -f k8s/k8s-rbac-ai.yaml kubectl apply -f k8s/k8s-resource-quotas-dev.yaml ``` -### Get the OIDC Issuer URL +### Get the OIDC Issuer URL & Tenant ID ```bash export AKS_OIDC_ISSUER="$(az aks show -n $CLUSTER_NAME -g rg-shipping-dronedelivery-${LOCATION} --query "oidcIssuerProfile.issuerUrl" -otsv)" +export TENANT_ID=$(az account show --query tenantId --output tsv) +``` + +### Create manage identity federations for microservices. +``` +# Setup managed identity for delivery microservice to trust your Kubernetes service account +az identity federated-credential create --name credential-for-delivery --identity-name uid-delivery --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:delivery-sa-v0.1.0 + +# Setup managed identity for package microservice to trust your Kubernetes service account +az identity federated-credential create --name credential-for-package --identity-name uid-package --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:package-sa-v0.1.0 + +# Setup your managed identity to trust your Kubernetes service account +az identity federated-credential create --name credential-for-workflow --identity-name uid-workflow --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:workflow-sa-v0.1.0 + +# Setup your managed identity to trust your Kubernetes service account +az identity federated-credential create --name credential-for-ingestion --identity-name uid-ingestion --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:ingestion-sa-v0.1.0 + +# Setup your managed identity to trust your Kubernetes service account +az identity federated-credential create --name credential-for-dronescheduler --identity-name uid-dronescheduler --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:dronescheduler-sa-v0.1.0 + ``` ### Collect details of managed ingress controller. @@ -186,7 +206,6 @@ openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ Extract resource details from deployment. ```bash -export TENANT_ID=$(az account show --query tenantId --output tsv) export COSMOSDB_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.deliveryCosmosDbName.value -o tsv) && \ export DATABASE_NAME="${COSMOSDB_NAME}-db" && \ export COLLECTION_NAME="${DATABASE_NAME}-col" && \ @@ -202,14 +221,11 @@ export DELIVERY_KEYVAULT_ID=$(az resource show -g rg-shipping-dronedelivery-${LO az role assignment create --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $DELIVERY_KEYVAULT_ID #wait for role assignment to finish. -sleep 60 +sleep 30 az keyvault secret set --name Delivery-Ingress-Tls-Key --vault-name $DELIVERY_KEYVAULT_NAME --value "$(cat ingestion-ingress-tls.key)" az keyvault secret set --name Delivery-Ingress-Tls-Crt --vault-name $DELIVERY_KEYVAULT_NAME --value "$(cat ingestion-ingress-tls.crt)" az role assignment delete --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $DELIVERY_KEYVAULT_ID -#Setup your managed identity to trust your Kubernetes service account -az identity federated-credential create --name credential-for-delivery --identity-name uid-delivery --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:delivery-sa-v0.1.0 - # Deploy the service helm package charts/delivery/ -u && \ helm install delivery-v0.1.0-dev delivery-v0.1.0.tgz \ @@ -250,9 +266,6 @@ export PACKAGE_ID_CLIENT_ID=$(az identity show -g rg-shipping-dronedelivery-${LO export COSMOSDB_COL_NAME_PACKAGE=packages -# Setup your managed identity to trust your Kubernetes service account -az identity federated-credential create --name credential-for-package --identity-name uid-package --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:package-sa-v0.1.0 - # Deploy service helm package charts/package/ -u && \ helm install package-v0.1.0-dev package-v0.1.0.tgz \ @@ -289,9 +302,6 @@ export WORKFLOW_NAMESPACE_NAME=$(az deployment group show -g rg-shipping-dronede export WORKFLOW_NAMESPACE_ENDPOINT=$(az servicebus namespace show -g rg-shipping-dronedelivery-${LOCATION} -n $WORKFLOW_NAMESPACE_NAME --query serviceBusEndpoint -o tsv) export WORKFLOW_NAMESPACE_SAS_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.workflowServiceAccessKeyName.value -o tsv) -# Setup your managed identity to trust your Kubernetes service account -az identity federated-credential create --name credential-for-workflow --identity-name uid-workflow --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:workflow-sa-v0.1.0 - # Deploy the service helm package charts/workflow/ -u && \ helm install workflow-v0.1.0-dev workflow-v0.1.0.tgz \ @@ -332,15 +342,13 @@ export INGESTION_ID_CLIENT_ID=$(az identity show -g rg-shipping-dronedelivery-${ export INGESTION_KEYVAULT_ID=$(az resource show -g rg-shipping-dronedelivery-${LOCATION} -n $INGESTION_KEYVAULT_NAME --resource-type 'Microsoft.KeyVault/vaults' --query id --output tsv) az role assignment create --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $INGESTION_KEYVAULT_ID -sleep 60 +# wait a while for the role propagation to finish. +sleep 30 az keyvault secret set --name Ingestion-Ingress-Tls-Key --vault-name $INGESTION_KEYVAULT_NAME --value "$(cat ingestion-ingress-tls.key)" az keyvault secret set --name Ingestion-Ingress-Tls-Crt --vault-name $INGESTION_KEYVAULT_NAME --value "$(cat ingestion-ingress-tls.crt)" az role assignment delete --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $INGESTION_KEYVAULT_ID -# Setup your managed identity to trust your Kubernetes service account -az identity federated-credential create --name credential-for-ingestion --identity-name uid-ingestion --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:ingestion-sa-v0.1.0 - # Deploy service helm package charts/ingestion/ -u && \ helm install ingestion-v0.1.0-dev ingestion-v0.1.0.tgz \ @@ -382,9 +390,6 @@ export DRONESCHEDULER_CLIENT_ID=$(az identity show -g rg-shipping-dronedelivery- export DATABASE_NAME="invoicing" && \ export COLLECTION_NAME="utilization" -# Setup your managed identity to trust your Kubernetes service account -az identity federated-credential create --name credential-for-dronescheduler --identity-name uid-dronescheduler --resource-group rg-shipping-dronedelivery-${LOCATION} --issuer ${AKS_OIDC_ISSUER} --subject system:serviceaccount:backend-dev:dronescheduler-sa-v0.1.0 - # Deploy the service helm package charts/dronescheduler/ -u && \ helm install dronescheduler-v0.1.0-dev dronescheduler-v0.1.0.tgz \ From 09bed7e59ac65d61d9cdf5fa18438b6fb59b076d Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Fri, 31 Jan 2025 15:27:03 +0300 Subject: [PATCH 41/44] Update deployment.md --- deployment.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/deployment.md b/deployment.md index 20772671..932e6e54 100644 --- a/deployment.md +++ b/deployment.md @@ -145,11 +145,12 @@ kubectl apply -f k8s/k8s-rbac-ai.yaml kubectl apply -f k8s/k8s-resource-quotas-dev.yaml ``` -### Get the OIDC Issuer URL & Tenant ID +### Get the OIDC Issuer URL, Tenant ID, and Object ID of the signed-in user. ```bash export AKS_OIDC_ISSUER="$(az aks show -n $CLUSTER_NAME -g rg-shipping-dronedelivery-${LOCATION} --query "oidcIssuerProfile.issuerUrl" -otsv)" export TENANT_ID=$(az account show --query tenantId --output tsv) +export SIGNED_IN_OBJECT_ID=$(az ad signed-in-user show --query 'id' -o tsv) ``` ### Create manage identity federations for microservices. @@ -212,12 +213,11 @@ export COLLECTION_NAME="${DATABASE_NAME}-col" && \ export DELIVERY_KEYVAULT_URI=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.deliveryKeyVaultUri.value -o tsv) && \ export DELIVERY_KEYVAULT_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.deliveryKeyVaultName.value -o tsv) && \ export DELIVERY_PRINCIPAL_CLIENT_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-delivery --query clientId -o tsv) +export DELIVERY_KEYVAULT_ID=$(az resource show -g rg-shipping-dronedelivery-${LOCATION} -n $DELIVERY_KEYVAULT_NAME --resource-type 'Microsoft.KeyVault/vaults' --query id --output tsv) # Create secrets # The current user is given permission to import secrets and then it is deleted right after the secret creation command is executed -export SIGNED_IN_OBJECT_ID=$(az ad signed-in-user show --query 'id' -o tsv) -export DELIVERY_KEYVAULT_ID=$(az resource show -g rg-shipping-dronedelivery-${LOCATION} -n $DELIVERY_KEYVAULT_NAME --resource-type 'Microsoft.KeyVault/vaults' --query id --output tsv) az role assignment create --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $DELIVERY_KEYVAULT_ID #wait for role assignment to finish. @@ -261,9 +261,6 @@ Extract resource details from deployment. ```bash export PACKAGE_KEYVAULT_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.packageKeyVaultName.value -o tsv) export PACKAGE_ID_CLIENT_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-package --query clientId -o tsv) - -# Deploy the Package service. - export COSMOSDB_COL_NAME_PACKAGE=packages # Deploy service @@ -337,9 +334,9 @@ export INGESTION_QUEUE_NAMESPACE=$(az deployment group show -g rg-shipping-drone export INGESTION_QUEUE_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.ingestionQueueName.value -o tsv) export INGESTION_KEYVAULT_NAME=$(az deployment group show -g rg-shipping-dronedelivery-${LOCATION} -n workload-stamp --query properties.outputs.ingestionKeyVaultName.value -o tsv) export INGESTION_ID_CLIENT_ID=$(az identity show -g rg-shipping-dronedelivery-${LOCATION} -n uid-ingestion --query clientId -o tsv) +export INGESTION_KEYVAULT_ID=$(az resource show -g rg-shipping-dronedelivery-${LOCATION} -n $INGESTION_KEYVAULT_NAME --resource-type 'Microsoft.KeyVault/vaults' --query id --output tsv) # The current user is given permission to import secrets and then it is deleted right after the secret creation command is executed -export INGESTION_KEYVAULT_ID=$(az resource show -g rg-shipping-dronedelivery-${LOCATION} -n $INGESTION_KEYVAULT_NAME --resource-type 'Microsoft.KeyVault/vaults' --query id --output tsv) az role assignment create --role 'Key Vault Secrets Officer' --assignee $SIGNED_IN_OBJECT_ID --scope $INGESTION_KEYVAULT_ID # wait a while for the role propagation to finish. From f3c74277d6e0e344c38cd41897e283748c6ad8f1 Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Fri, 31 Jan 2025 15:29:39 +0300 Subject: [PATCH 42/44] Update deployment.md --- deployment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment.md b/deployment.md index 932e6e54..2e87ff9a 100644 --- a/deployment.md +++ b/deployment.md @@ -97,7 +97,7 @@ az acr build -r $ACR_NAME -f ./workload/src/shipping/dronescheduler/Dockerfile - az acr build -r $ACR_NAME -t $ACR_SERVER/package:0.1.0 ./workload/src/shipping/package/. ``` -## Deploy the managed cluster and all related resources +## Deploy the managed cluster and related resources ```bash From 5edd0807e1dbaf843fa63d76960f8efb209ab09f Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Fri, 31 Jan 2025 17:18:35 +0300 Subject: [PATCH 43/44] Update deployment.md --- deployment.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deployment.md b/deployment.md index 2e87ff9a..73572d25 100644 --- a/deployment.md +++ b/deployment.md @@ -412,6 +412,11 @@ helm install dronescheduler-v0.1.0-dev dronescheduler-v0.1.0.tgz \ helm status dronescheduler-v0.1.0-dev --namespace backend-dev ``` +## Verify that all the microservice pods are in running state. +``` +kubectl get pods -n backend-dev --watch +``` + ## Validate the application is running You can send delivery requests and check their statuses using curl. From e5516546bd69a62213c647a1205c492e4171ed6a Mon Sep 17 00:00:00 2001 From: Francis Simy Nazareth <73844173+francisnazareth@users.noreply.github.com> Date: Fri, 31 Jan 2025 17:19:50 +0300 Subject: [PATCH 44/44] Update deployment.md --- deployment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment.md b/deployment.md index 73572d25..e9c3acf7 100644 --- a/deployment.md +++ b/deployment.md @@ -412,7 +412,7 @@ helm install dronescheduler-v0.1.0-dev dronescheduler-v0.1.0.tgz \ helm status dronescheduler-v0.1.0-dev --namespace backend-dev ``` -## Verify that all the microservice pods are in running state. +## Verify that all the microservice pods are ready and are in running state. ``` kubectl get pods -n backend-dev --watch ```