ci: cns patch testing in release pipeline

.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml

@@ -223,7 +223,9 @@ stages:
       - job: deploy_pods
         condition: and( and( not(canceled()), not(failed()) ), or( contains(variables.CONTROL_SCENARIO, 'scaleTest') , contains(variables.CONTROL_SCENARIO, 'all') ) )
         displayName: "Scale Test"
-        dependsOn: deploy_cilium_components
+        dependsOn: 
+          - deploy_cilium_components
+          - deploy_cns_and_ipam
         steps:
           - template: ../load-test-templates/pod-deployment-template.yaml
             parameters:

.pipelines/cni/pipeline.yaml

@@ -210,6 +210,7 @@ stages:
       osSkuWin: 'Windows2022'
       scaleup: ${SCALEUP_WIN}
       iterations: ${ITERATIONS_WIN}
+      upgradeScenario: false
 
   - template: singletenancy/cniv2-template.yaml
     parameters:
@@ -226,6 +227,7 @@ stages:
       osSkuWin: 'Windows2022'
       scaleup: ${SCALEUP_WIN}
       iterations: ${ITERATIONS_WIN}
+      upgradeScenario: false
 
 ## Linux E2E
   - template: singletenancy/cniv1-template.yaml
@@ -250,6 +252,7 @@ stages:
       arch: amd64
       scaleup: ${SCALEUP_LINUX}
       iterations: ${ITERATIONS_LINUX}
+      upgradeScenario: false
 
   - template: singletenancy/cniv2-template.yaml
     parameters:
@@ -261,6 +264,7 @@ stages:
       arch: amd64
       scaleup: ${SCALEUP_LINUX}
       iterations: ${ITERATIONS_LINUX}
+      upgradeScenario: false
 
   - template: singletenancy/cniv2-template.yaml
     parameters:
@@ -273,6 +277,7 @@ stages:
       osSKU: AzureLinux
       scaleup: ${SCALEUP_LINUX}
       iterations: ${ITERATIONS_LINUX}
+      upgradeScenario: false
 
   - template: singletenancy/cniv2-template.yaml
     parameters:
@@ -284,6 +289,7 @@ stages:
       arch: arm64
       scaleup: ${SCALEUP_LINUX}
       iterations: ${ITERATIONS_LINUX}
+      upgradeScenario: false
 
   - template: singletenancy/cniv2-template.yaml
     parameters:
@@ -295,6 +301,7 @@ stages:
       arch: amd64
       scaleup: ${SCALEUP_LINUX}
       iterations: ${ITERATIONS_LINUX}
+      upgradeScenario: false
 
 ## Cilium E2E
   - template: cilium/cilium-overlay-load-test-template.yaml
@@ -404,7 +411,7 @@ stages:
       upgradeScenario: false
 
 
-  ## If upgradeScenario is set, redeploy new IPAM version to existing clusters and run tests
+  ## If upgradeScenario is set, redeploy new IPAM/CNS version to existing clusters and run tests
   - ${{if eq(parameters.upgradeScenario, true)}}:
     - template: cilium/cilium-overlay-load-test-template.yaml
       parameters:
@@ -520,6 +527,108 @@ stages:
         cni: "cilium_dualstack"
         upgradeScenario: true
         dependsOn: cilium_ds_rdma
+
+    - template: singletenancy/cniv2-template.yaml
+      parameters:
+        name: linux_podsubnet_upgrade
+        clusterType: swift-byocni-up
+        clusterName: "linux-podsub"
+        nodeCount: ${NODE_COUNT_LINUX}
+        vmSize: ${VM_SIZE}
+        arch: amd64
+        scaleup: ${SCALEUP_LINUX}
+        iterations: ${ITERATIONS_LINUX}
+        upgradeScenario: true
+        dependsOn: linux_podsubnet
+
+    - template: singletenancy/cniv2-template.yaml
+      parameters:
+        name: linux_overlay_upgrade
+        clusterType: overlay-byocni-up
+        clusterName: "linux-over"
+        nodeCount: ${NODE_COUNT_LINUX}
+        vmSize: ${VM_SIZE}
+        arch: amd64
+        scaleup: ${SCALEUP_LINUX}
+        iterations: ${ITERATIONS_LINUX}
+        upgradeScenario: true
+        dependsOn: linux_overlay
+
+    - template: singletenancy/cniv2-template.yaml
+      parameters:
+        name: mariner_linux_overlay_upgrade
+        clusterType: overlay-byocni-up
+        clusterName: "mariner-over"
+        nodeCount: ${NODE_COUNT_LINUX}
+        vmSize:  ${VM_SIZE}
+        arch: amd64
+        osSKU: AzureLinux
+        scaleup: ${SCALEUP_LINUX}
+        iterations: ${ITERATIONS_LINUX}
+        upgradeScenario: true
+        dependsOn: mariner_linux_overlay
+
+    - template: singletenancy/cniv2-template.yaml
+      parameters:
+        name: arm_linux_overlay_upgrade
+        clusterType: overlay-byocni-up
+        clusterName: "arm-over"
+        nodeCount: ${NODE_COUNT_LINUX}
+        vmSize: Standard_D8ps_v5
+        arch: arm64
+        scaleup: ${SCALEUP_LINUX}
+        iterations: ${ITERATIONS_LINUX}
+        upgradeScenario: true
+        dependsOn: arm_linux_overlay
+
+    - template: singletenancy/cniv2-template.yaml
+      parameters:
+        name: rdma_linux_overlay_upgrade
+        clusterType: overlay-byocni-up
+        clusterName: "rdma-over"
+        nodeCount: 2
+        vmSize: Standard_HC44-16rs
+        arch: amd64
+        scaleup: ${SCALEUP_LINUX}
+        iterations: ${ITERATIONS_LINUX}
+        upgradeScenario: true
+        dependsOn: rdma_linux_overlay
+
+    - template: singletenancy/cniv2-template.yaml
+      parameters:
+        name: windows_podsubnet_upgrade
+        clusterType: swift-byocni-up
+        clusterName: w22-podsub
+        nodeCount: ${NODE_COUNT_WINCLUSTER_SYSTEMPOOL}
+        nodeCountWin: ${NODE_COUNT_WIN}
+        vmSize: ${VM_SIZE_WINCLUSTER_SYSTEMPOOL}
+        vmSizeWin: ${VM_SIZE_WIN}
+        arch: amd64
+        os: windows
+        os_version: 'ltsc2022'
+        osSkuWin: 'Windows2022'
+        scaleup: ${SCALEUP_WIN}
+        iterations: ${ITERATIONS_WIN}
+        upgradeScenario: true
+        dependsOn: windows_podsubnet_HNS
+
+    - template: singletenancy/cniv2-template.yaml
+      parameters:
+        name: windows_overlay_upgrade
+        clusterType: overlay-byocni-up
+        clusterName: w22-over
+        nodeCount: ${NODE_COUNT_WINCLUSTER_SYSTEMPOOL}
+        nodeCountWin: ${NODE_COUNT_WIN}
+        vmSize: ${VM_SIZE_WINCLUSTER_SYSTEMPOOL}
+        vmSizeWin: ${VM_SIZE_WIN}
+        arch: amd64
+        os: windows
+        os_version: 'ltsc2022'
+        osSkuWin: 'Windows2022'
+        scaleup: ${SCALEUP_WIN}
+        iterations: ${ITERATIONS_WIN}
+        upgradeScenario: true
+        dependsOn: windows_overlay_HNS
 
   - stage: delete_resources
     displayName: "Delete Resources"
@@ -556,6 +665,15 @@ stages:
         - cilium_ds_arm_upgrade
         - cilium_ds_mariner_upgrade
         - cilium_ds_rdma_upgrade
+        - linux_podsubnet_upgrade
+        - linux_overlay_upgrade
+        - mariner_linux_overlay_upgrade
+        - arm_linux_overlay_upgrade
+        - rdma_linux_overlay_upgrade
+        - windows_podsubnet_upgrade
+        - windows_overlay_upgrade
+        - windows_podsubnet_upgrade_HNS
+        - windows_overlay_upgrade_HNS
     variables:
       commitID: $[ stagedependencies.setup.env.outputs['SetEnvVars.commitID'] ]
     jobs:

.pipelines/cni/singletenancy/cniv2-template.yaml

@@ -12,6 +12,7 @@ parameters:
   iterations: 3
   osSKU: Ubuntu
   osSkuWin: Windows2022
+  upgradeScenario: ""
 
 # CNIv2
 # + Should be able to scale up/down the pods successfully certain number of times.
@@ -28,7 +29,7 @@ parameters:
 # Previous job has reported Succeeded. Previous job is currently setup which controls variable assignment and we are dependent on its success.
 # CONTROL_OS either contains 'linux' or 'all' and CONTROL_CNI either contains 'cniv1' or 'all'. Both must be true and are not case sensitive
 stages:
-  - ${{ if eq(parameters.os, 'linux') }}:
+  - ${{ if and(eq(parameters.os, 'linux'), eq(parameters.upgradeScenario, false)) }}:
     - stage: create_${{ parameters.name }}
       condition: and( succeeded(), and( or( contains(variables.CONTROL_CNI, 'cniv2'), contains(variables.CONTROL_CNI, 'all') ), or( contains(variables.CONTROL_OS, 'linux'), contains(variables.CONTROL_OS, 'all') ) ) )
       variables:
@@ -61,7 +62,7 @@ stages:
 # Condition confirms that:
 # Previous job has reported Succeeded. Previous job is currently setup which controls variable assignment and we are dependent on its success.
 # CONTROL_OS either contains 'windows' or 'all' and CONTROL_CNI either contains 'cniv2' or 'all'. Both must be true and are not case sensitive
-  - ${{ if eq(parameters.os, 'windows') }}:
+  - ${{ if and(eq(parameters.os, 'windows'), eq(parameters.upgradeScenario, false)) }}:
     - stage: create_${{ parameters.name }}
       condition: and( succeeded(), and( or( contains(variables.CONTROL_CNI, 'cniv2'), contains(variables.CONTROL_CNI, 'all') ), or( contains(variables.CONTROL_OS, 'windows'), contains(variables.CONTROL_OS, 'all') ) ) )
       variables:
@@ -116,7 +117,12 @@ stages:
     pool:
       name: $(BUILD_POOL_NAME_DEFAULT)
     dependsOn:
-      - create_${{ parameters.name }}
+      - ${{ if and(eq(parameters.upgradeScenario, true), ne(parameters.dependsOn, '')) }}:
+        - ${{ parameters.dependsOn }}
+      - ${{ if and(eq(parameters.upgradeScenario, false), eq(parameters.os, 'linux')) }}:
+        - create_${{ parameters.name }}
+      - ${{ if and(eq(parameters.upgradeScenario, false), eq(parameters.os, 'windows')) }}:
+        - create_${{ parameters.name }}
       - publish
       - setup
     displayName: "CNIv2 Test - ${{ parameters.name }}"
@@ -136,11 +142,34 @@ stages:
                   make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID)
                   kubectl cluster-info
                   kubectl get po -owide -A
+                  if [ ${{parameters.upgradeScenario}} == "true" ]; then
+                    if [ -z "$UPGRADE_CNS_VERSION" ]
+                    then
+                      echo "UPGRADE_CNS_VERSION is not set, using default value"
+                      CNS=$(make cns-version)
+                    else
+                      CNS=$UPGRADE_CNS_VERSION
+                      echo "UPGRADE_CNS_VERSION is set to $CNS"
+                    fi
+                    IPAM_IMAGE_REPO=$UPGRADE_IPAM_IMAGE_REPO
+                    CNS_IMAGE_REPO=$UPGRADE_CNS_IMAGE_REPO
+                  else
+                    if [ -z "$TEST_CNS_VERSION" ]
+                    then
+                      echo "TEST_CNS_VERSION is not set, using default value"
+                      CNS=$(make cns-version)
+                    else
+                      CNS=$TEST_CNS_VERSION
+                      echo "TEST_CNS_VERSION is set to $CNS"
+                    fi
+                  fi
+
                   if [ "${{parameters.os}}" == "windows" ]; then
-                    sudo -E env "PATH=$PATH" make test-load CNS_ONLY=true CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO)
+                    sudo -E env "PATH=$PATH" make test-load CNS_ONLY=true CNS_VERSION=${CNS} CNI_VERSION=$(make cni-version) INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO)
                   else
-                    sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO)
+                    sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=${CNS} CNI_VERSION=$(make cni-version) INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO)
                   fi
+
               name: "overlaye2e"
               displayName: "Overlay Integration"
           - ${{ if contains(parameters.clusterType, 'swift') }}:
@@ -155,10 +184,31 @@ stages:
                   make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID)
                   kubectl cluster-info
                   kubectl get po -owide -A
+                  if [ ${{parameters.upgradeScenario}} == "true" ]; then
+                    if [ -z "$UPGRADE_CNS_VERSION" ]
+                    then
+                      echo "UPGRADE_CNS_VERSION is not set, using default value"
+                      CNS=$(make cns-version)
+                    else
+                      CNS=$UPGRADE_CNS_VERSION
+                      echo "UPGRADE_CNS_VERSION is set to $CNS"
+                    fi
+                    IPAM_IMAGE_REPO=$UPGRADE_IPAM_IMAGE_REPO
+                    CNS_IMAGE_REPO=$UPGRADE_CNS_IMAGE_REPO
+                  else
+                    if [ -z "$TEST_CNS_VERSION" ]
+                    then
+                      echo "TEST_CNS_VERSION is not set, using default value"
+                      CNS=$(make cns-version)
+                    else
+                      CNS=$TEST_CNS_VERSION
+                      echo "TEST_CNS_VERSION is set to $CNS"
+                    fi
+                  fi
                   if [ "${{parameters.os}}" == "windows" ]; then
-                    sudo -E env "PATH=$PATH" make test-load CNS_ONLY=true CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) INSTALL_CNS=true INSTALL_AZURE_VNET=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO)
+                    sudo -E env "PATH=$PATH" make test-load CNS_ONLY=true CNS_VERSION=${CNS} CNI_VERSION=$(make cni-version) INSTALL_CNS=true INSTALL_AZURE_VNET=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO)
                   else
-                    sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) INSTALL_CNS=true INSTALL_AZURE_VNET=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO)
+                    sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=${CNS} CNI_VERSION=$(make cni-version) INSTALL_CNS=true INSTALL_AZURE_VNET=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO)
                   fi
               name: "swifte2e"
               displayName: "Swift Integration"
@@ -195,6 +245,8 @@ stages:
           tag: $(npmVersion)
           dependsOn: integration
           continueOnError: true
+          ${{ if eq(parameters.upgradeScenario, 'true') }}:
+            logType: upgradeNpmLogs
       - job: deploy_pods
         condition: and( and( not(canceled()), not(failed()) ), or( contains(variables.CONTROL_SCENARIO, 'scaleTest') , contains(variables.CONTROL_SCENARIO, 'all') ) )
         displayName: "Scale Test"
@@ -208,6 +260,8 @@ stages:
               iterations: ${{ parameters.iterations }}
               nodeCount: $(nodeCount)
               cni: cniv2
+              ${{ if eq(parameters.upgradeScenario, 'true') }}:
+                logType: upgradeScaleTest
           - template: ../load-test-templates/validate-state-template.yaml
             parameters:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
@@ -225,6 +279,8 @@ stages:
               os: ${{ parameters.os }}
               cni: cniv2
               region: $(location)
+              ${{ if eq(parameters.upgradeScenario, 'true') }}:
+                logType: upgradeRestartNode
           - template: ../load-test-templates/validate-state-template.yaml
             parameters:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
@@ -243,6 +299,8 @@ stages:
               cni: cniv2
               scaleup: ${{ parameters.scaleup }}
               nodeCount: ${{ parameters.nodeCount }}
+              ${{ if eq(parameters.upgradeScenario, 'true') }}:
+                logType: upgradeRestartCNS
       - job: recover
         condition: and( not(canceled()), not(failed()) )
         displayName: "Recover Resources"
@@ -309,7 +367,10 @@ stages:
       dependsOn:
         - ${{ parameters.name }}
         - setup
-      displayName: "HNS Test - ${{ parameters.clusterName }}"
+      ${{ if eq(parameters.upgradeScenario, true) }}:
+        displayName: "HNS Test w/ Upgrade - ${{ parameters.name }}"
+      ${{ else }}:
+        displayName: "HNS Test - ${{ parameters.name }}"
       jobs:
         - job: restart_hns
           displayName: "Restart and Validate HNS"

.pipelines/npm/npm-cni-integration-test.yaml

@@ -6,6 +6,7 @@ parameters:
   dependsOn: ""
   tag: ""
   continueOnError: false
+  logType: "npmLogs"
 
 # Condition confirms that:
 # Pipeline has not been canceled and that the previous job has reports anything other than failure(Succeeded, SuccededWithIssues, Skipped). Previous job is declared by dependsOn: ${{ parameters.dependsOn }}
@@ -134,7 +135,7 @@ jobs:
             condition: always()
 
       - bash: |
-          npmLogs=$(System.DefaultWorkingDirectory)/${{ parameters.clusterName }}_npmLogs_Attempt_#$(System.StageAttempt)
+          npmLogs=$(System.DefaultWorkingDirectory)/${{ parameters.clusterName }}_${{ parameters.logType }}_Attempt_#$(System.StageAttempt)
           mkdir -p $npmLogs
 
           kubectl get pods -n kube-system | grep npm
@@ -147,8 +148,8 @@ jobs:
         retryCountOnTaskFailure: 3
         condition: always()
 
-      - publish: $(System.DefaultWorkingDirectory)/${{ parameters.clusterName }}_npmLogs_Attempt_#$(System.StageAttempt)
+      - publish: $(System.DefaultWorkingDirectory)/${{ parameters.clusterName }}_${{ parameters.logType }}_Attempt_#$(System.StageAttempt)
         condition: always()
-        artifact: ${{ parameters.clusterName }}_npmLogs_Attempt_#$(System.StageAttempt)
+        artifact: ${{ parameters.clusterName }}_${{ parameters.logType }}_Attempt_#$(System.StageAttempt)
         name: npmLogs
         displayName: Publish NPM pod logs