diff --git a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml index 1f808d74e7..c0789f590f 100644 --- a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml +++ b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml @@ -223,7 +223,9 @@ stages: - job: deploy_pods condition: and( and( not(canceled()), not(failed()) ), or( contains(variables.CONTROL_SCENARIO, 'scaleTest') , contains(variables.CONTROL_SCENARIO, 'all') ) ) displayName: "Scale Test" - dependsOn: deploy_cilium_components + dependsOn: + - deploy_cilium_components + - deploy_cns_and_ipam steps: - template: ../load-test-templates/pod-deployment-template.yaml parameters: diff --git a/.pipelines/cni/pipeline.yaml b/.pipelines/cni/pipeline.yaml index 25b8caa62f..4947b296d2 100644 --- a/.pipelines/cni/pipeline.yaml +++ b/.pipelines/cni/pipeline.yaml @@ -210,6 +210,7 @@ stages: osSkuWin: 'Windows2022' scaleup: ${SCALEUP_WIN} iterations: ${ITERATIONS_WIN} + upgradeScenario: false - template: singletenancy/cniv2-template.yaml parameters: @@ -226,6 +227,7 @@ stages: osSkuWin: 'Windows2022' scaleup: ${SCALEUP_WIN} iterations: ${ITERATIONS_WIN} + upgradeScenario: false ## Linux E2E - template: singletenancy/cniv1-template.yaml @@ -250,6 +252,7 @@ stages: arch: amd64 scaleup: ${SCALEUP_LINUX} iterations: ${ITERATIONS_LINUX} + upgradeScenario: false - template: singletenancy/cniv2-template.yaml parameters: @@ -261,6 +264,7 @@ stages: arch: amd64 scaleup: ${SCALEUP_LINUX} iterations: ${ITERATIONS_LINUX} + upgradeScenario: false - template: singletenancy/cniv2-template.yaml parameters: @@ -273,6 +277,7 @@ stages: osSKU: AzureLinux scaleup: ${SCALEUP_LINUX} iterations: ${ITERATIONS_LINUX} + upgradeScenario: false - template: singletenancy/cniv2-template.yaml parameters: @@ -284,6 +289,7 @@ stages: arch: arm64 scaleup: ${SCALEUP_LINUX} iterations: ${ITERATIONS_LINUX} + upgradeScenario: false - template: singletenancy/cniv2-template.yaml parameters: @@ -295,6 +301,7 @@ stages: arch: amd64 scaleup: ${SCALEUP_LINUX} iterations: ${ITERATIONS_LINUX} + upgradeScenario: false ## Cilium E2E - template: cilium/cilium-overlay-load-test-template.yaml @@ -404,7 +411,7 @@ stages: upgradeScenario: false - ## If upgradeScenario is set, redeploy new IPAM version to existing clusters and run tests + ## If upgradeScenario is set, redeploy new IPAM/CNS version to existing clusters and run tests - ${{if eq(parameters.upgradeScenario, true)}}: - template: cilium/cilium-overlay-load-test-template.yaml parameters: @@ -520,6 +527,108 @@ stages: cni: "cilium_dualstack" upgradeScenario: true dependsOn: cilium_ds_rdma + + - template: singletenancy/cniv2-template.yaml + parameters: + name: linux_podsubnet_upgrade + clusterType: swift-byocni-up + clusterName: "linux-podsub" + nodeCount: ${NODE_COUNT_LINUX} + vmSize: ${VM_SIZE} + arch: amd64 + scaleup: ${SCALEUP_LINUX} + iterations: ${ITERATIONS_LINUX} + upgradeScenario: true + dependsOn: linux_podsubnet + + - template: singletenancy/cniv2-template.yaml + parameters: + name: linux_overlay_upgrade + clusterType: overlay-byocni-up + clusterName: "linux-over" + nodeCount: ${NODE_COUNT_LINUX} + vmSize: ${VM_SIZE} + arch: amd64 + scaleup: ${SCALEUP_LINUX} + iterations: ${ITERATIONS_LINUX} + upgradeScenario: true + dependsOn: linux_overlay + + - template: singletenancy/cniv2-template.yaml + parameters: + name: mariner_linux_overlay_upgrade + clusterType: overlay-byocni-up + clusterName: "mariner-over" + nodeCount: ${NODE_COUNT_LINUX} + vmSize: ${VM_SIZE} + arch: amd64 + osSKU: AzureLinux + scaleup: ${SCALEUP_LINUX} + iterations: ${ITERATIONS_LINUX} + upgradeScenario: true + dependsOn: mariner_linux_overlay + + - template: singletenancy/cniv2-template.yaml + parameters: + name: arm_linux_overlay_upgrade + clusterType: overlay-byocni-up + clusterName: "arm-over" + nodeCount: ${NODE_COUNT_LINUX} + vmSize: Standard_D8ps_v5 + arch: arm64 + scaleup: ${SCALEUP_LINUX} + iterations: ${ITERATIONS_LINUX} + upgradeScenario: true + dependsOn: arm_linux_overlay + + - template: singletenancy/cniv2-template.yaml + parameters: + name: rdma_linux_overlay_upgrade + clusterType: overlay-byocni-up + clusterName: "rdma-over" + nodeCount: 2 + vmSize: Standard_HC44-16rs + arch: amd64 + scaleup: ${SCALEUP_LINUX} + iterations: ${ITERATIONS_LINUX} + upgradeScenario: true + dependsOn: rdma_linux_overlay + + - template: singletenancy/cniv2-template.yaml + parameters: + name: windows_podsubnet_upgrade + clusterType: swift-byocni-up + clusterName: w22-podsub + nodeCount: ${NODE_COUNT_WINCLUSTER_SYSTEMPOOL} + nodeCountWin: ${NODE_COUNT_WIN} + vmSize: ${VM_SIZE_WINCLUSTER_SYSTEMPOOL} + vmSizeWin: ${VM_SIZE_WIN} + arch: amd64 + os: windows + os_version: 'ltsc2022' + osSkuWin: 'Windows2022' + scaleup: ${SCALEUP_WIN} + iterations: ${ITERATIONS_WIN} + upgradeScenario: true + dependsOn: windows_podsubnet_HNS + + - template: singletenancy/cniv2-template.yaml + parameters: + name: windows_overlay_upgrade + clusterType: overlay-byocni-up + clusterName: w22-over + nodeCount: ${NODE_COUNT_WINCLUSTER_SYSTEMPOOL} + nodeCountWin: ${NODE_COUNT_WIN} + vmSize: ${VM_SIZE_WINCLUSTER_SYSTEMPOOL} + vmSizeWin: ${VM_SIZE_WIN} + arch: amd64 + os: windows + os_version: 'ltsc2022' + osSkuWin: 'Windows2022' + scaleup: ${SCALEUP_WIN} + iterations: ${ITERATIONS_WIN} + upgradeScenario: true + dependsOn: windows_overlay_HNS - stage: delete_resources displayName: "Delete Resources" @@ -556,6 +665,15 @@ stages: - cilium_ds_arm_upgrade - cilium_ds_mariner_upgrade - cilium_ds_rdma_upgrade + - linux_podsubnet_upgrade + - linux_overlay_upgrade + - mariner_linux_overlay_upgrade + - arm_linux_overlay_upgrade + - rdma_linux_overlay_upgrade + - windows_podsubnet_upgrade + - windows_overlay_upgrade + - windows_podsubnet_upgrade_HNS + - windows_overlay_upgrade_HNS variables: commitID: $[ stagedependencies.setup.env.outputs['SetEnvVars.commitID'] ] jobs: diff --git a/.pipelines/cni/singletenancy/cniv2-template.yaml b/.pipelines/cni/singletenancy/cniv2-template.yaml index d040e2569b..adc0d322fa 100644 --- a/.pipelines/cni/singletenancy/cniv2-template.yaml +++ b/.pipelines/cni/singletenancy/cniv2-template.yaml @@ -12,6 +12,7 @@ parameters: iterations: 3 osSKU: Ubuntu osSkuWin: Windows2022 + upgradeScenario: "" # CNIv2 # + Should be able to scale up/down the pods successfully certain number of times. @@ -28,7 +29,7 @@ parameters: # Previous job has reported Succeeded. Previous job is currently setup which controls variable assignment and we are dependent on its success. # CONTROL_OS either contains 'linux' or 'all' and CONTROL_CNI either contains 'cniv1' or 'all'. Both must be true and are not case sensitive stages: - - ${{ if eq(parameters.os, 'linux') }}: + - ${{ if and(eq(parameters.os, 'linux'), eq(parameters.upgradeScenario, false)) }}: - stage: create_${{ parameters.name }} condition: and( succeeded(), and( or( contains(variables.CONTROL_CNI, 'cniv2'), contains(variables.CONTROL_CNI, 'all') ), or( contains(variables.CONTROL_OS, 'linux'), contains(variables.CONTROL_OS, 'all') ) ) ) variables: @@ -61,7 +62,7 @@ stages: # Condition confirms that: # Previous job has reported Succeeded. Previous job is currently setup which controls variable assignment and we are dependent on its success. # CONTROL_OS either contains 'windows' or 'all' and CONTROL_CNI either contains 'cniv2' or 'all'. Both must be true and are not case sensitive - - ${{ if eq(parameters.os, 'windows') }}: + - ${{ if and(eq(parameters.os, 'windows'), eq(parameters.upgradeScenario, false)) }}: - stage: create_${{ parameters.name }} condition: and( succeeded(), and( or( contains(variables.CONTROL_CNI, 'cniv2'), contains(variables.CONTROL_CNI, 'all') ), or( contains(variables.CONTROL_OS, 'windows'), contains(variables.CONTROL_OS, 'all') ) ) ) variables: @@ -116,7 +117,12 @@ stages: pool: name: $(BUILD_POOL_NAME_DEFAULT) dependsOn: - - create_${{ parameters.name }} + - ${{ if and(eq(parameters.upgradeScenario, true), ne(parameters.dependsOn, '')) }}: + - ${{ parameters.dependsOn }} + - ${{ if and(eq(parameters.upgradeScenario, false), eq(parameters.os, 'linux')) }}: + - create_${{ parameters.name }} + - ${{ if and(eq(parameters.upgradeScenario, false), eq(parameters.os, 'windows')) }}: + - create_${{ parameters.name }} - publish - setup displayName: "CNIv2 Test - ${{ parameters.name }}" @@ -136,11 +142,34 @@ stages: make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) kubectl cluster-info kubectl get po -owide -A + if [ ${{parameters.upgradeScenario}} == "true" ]; then + if [ -z "$UPGRADE_CNS_VERSION" ] + then + echo "UPGRADE_CNS_VERSION is not set, using default value" + CNS=$(make cns-version) + else + CNS=$UPGRADE_CNS_VERSION + echo "UPGRADE_CNS_VERSION is set to $CNS" + fi + IPAM_IMAGE_REPO=$UPGRADE_IPAM_IMAGE_REPO + CNS_IMAGE_REPO=$UPGRADE_CNS_IMAGE_REPO + else + if [ -z "$TEST_CNS_VERSION" ] + then + echo "TEST_CNS_VERSION is not set, using default value" + CNS=$(make cns-version) + else + CNS=$TEST_CNS_VERSION + echo "TEST_CNS_VERSION is set to $CNS" + fi + fi + if [ "${{parameters.os}}" == "windows" ]; then - sudo -E env "PATH=$PATH" make test-load CNS_ONLY=true CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) + sudo -E env "PATH=$PATH" make test-load CNS_ONLY=true CNS_VERSION=${CNS} CNI_VERSION=$(make cni-version) INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) else - sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) + sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=${CNS} CNI_VERSION=$(make cni-version) INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) fi + name: "overlaye2e" displayName: "Overlay Integration" - ${{ if contains(parameters.clusterType, 'swift') }}: @@ -155,10 +184,31 @@ stages: make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) kubectl cluster-info kubectl get po -owide -A + if [ ${{parameters.upgradeScenario}} == "true" ]; then + if [ -z "$UPGRADE_CNS_VERSION" ] + then + echo "UPGRADE_CNS_VERSION is not set, using default value" + CNS=$(make cns-version) + else + CNS=$UPGRADE_CNS_VERSION + echo "UPGRADE_CNS_VERSION is set to $CNS" + fi + IPAM_IMAGE_REPO=$UPGRADE_IPAM_IMAGE_REPO + CNS_IMAGE_REPO=$UPGRADE_CNS_IMAGE_REPO + else + if [ -z "$TEST_CNS_VERSION" ] + then + echo "TEST_CNS_VERSION is not set, using default value" + CNS=$(make cns-version) + else + CNS=$TEST_CNS_VERSION + echo "TEST_CNS_VERSION is set to $CNS" + fi + fi if [ "${{parameters.os}}" == "windows" ]; then - sudo -E env "PATH=$PATH" make test-load CNS_ONLY=true CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) INSTALL_CNS=true INSTALL_AZURE_VNET=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) + sudo -E env "PATH=$PATH" make test-load CNS_ONLY=true CNS_VERSION=${CNS} CNI_VERSION=$(make cni-version) INSTALL_CNS=true INSTALL_AZURE_VNET=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) else - sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) INSTALL_CNS=true INSTALL_AZURE_VNET=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) + sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=${CNS} CNI_VERSION=$(make cni-version) INSTALL_CNS=true INSTALL_AZURE_VNET=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) fi name: "swifte2e" displayName: "Swift Integration" @@ -195,6 +245,8 @@ stages: tag: $(npmVersion) dependsOn: integration continueOnError: true + ${{ if eq(parameters.upgradeScenario, 'true') }}: + logType: upgradeNpmLogs - job: deploy_pods condition: and( and( not(canceled()), not(failed()) ), or( contains(variables.CONTROL_SCENARIO, 'scaleTest') , contains(variables.CONTROL_SCENARIO, 'all') ) ) displayName: "Scale Test" @@ -208,6 +260,8 @@ stages: iterations: ${{ parameters.iterations }} nodeCount: $(nodeCount) cni: cniv2 + ${{ if eq(parameters.upgradeScenario, 'true') }}: + logType: upgradeScaleTest - template: ../load-test-templates/validate-state-template.yaml parameters: clusterName: ${{ parameters.clusterName }}-$(commitID) @@ -225,6 +279,8 @@ stages: os: ${{ parameters.os }} cni: cniv2 region: $(location) + ${{ if eq(parameters.upgradeScenario, 'true') }}: + logType: upgradeRestartNode - template: ../load-test-templates/validate-state-template.yaml parameters: clusterName: ${{ parameters.clusterName }}-$(commitID) @@ -243,6 +299,8 @@ stages: cni: cniv2 scaleup: ${{ parameters.scaleup }} nodeCount: ${{ parameters.nodeCount }} + ${{ if eq(parameters.upgradeScenario, 'true') }}: + logType: upgradeRestartCNS - job: recover condition: and( not(canceled()), not(failed()) ) displayName: "Recover Resources" @@ -309,7 +367,10 @@ stages: dependsOn: - ${{ parameters.name }} - setup - displayName: "HNS Test - ${{ parameters.clusterName }}" + ${{ if eq(parameters.upgradeScenario, true) }}: + displayName: "HNS Test w/ Upgrade - ${{ parameters.name }}" + ${{ else }}: + displayName: "HNS Test - ${{ parameters.name }}" jobs: - job: restart_hns displayName: "Restart and Validate HNS" diff --git a/.pipelines/npm/npm-cni-integration-test.yaml b/.pipelines/npm/npm-cni-integration-test.yaml index 4660ced284..6d598b9a8f 100644 --- a/.pipelines/npm/npm-cni-integration-test.yaml +++ b/.pipelines/npm/npm-cni-integration-test.yaml @@ -6,6 +6,7 @@ parameters: dependsOn: "" tag: "" continueOnError: false + logType: "npmLogs" # Condition confirms that: # Pipeline has not been canceled and that the previous job has reports anything other than failure(Succeeded, SuccededWithIssues, Skipped). Previous job is declared by dependsOn: ${{ parameters.dependsOn }} @@ -134,7 +135,7 @@ jobs: condition: always() - bash: | - npmLogs=$(System.DefaultWorkingDirectory)/${{ parameters.clusterName }}_npmLogs_Attempt_#$(System.StageAttempt) + npmLogs=$(System.DefaultWorkingDirectory)/${{ parameters.clusterName }}_${{ parameters.logType }}_Attempt_#$(System.StageAttempt) mkdir -p $npmLogs kubectl get pods -n kube-system | grep npm @@ -147,8 +148,8 @@ jobs: retryCountOnTaskFailure: 3 condition: always() - - publish: $(System.DefaultWorkingDirectory)/${{ parameters.clusterName }}_npmLogs_Attempt_#$(System.StageAttempt) + - publish: $(System.DefaultWorkingDirectory)/${{ parameters.clusterName }}_${{ parameters.logType }}_Attempt_#$(System.StageAttempt) condition: always() - artifact: ${{ parameters.clusterName }}_npmLogs_Attempt_#$(System.StageAttempt) + artifact: ${{ parameters.clusterName }}_${{ parameters.logType }}_Attempt_#$(System.StageAttempt) name: npmLogs displayName: Publish NPM pod logs