From e57d37f1190cf05ff2a3159e45fea6991b55ba88 Mon Sep 17 00:00:00 2001 From: Shyju Krishnankutty Date: Wed, 5 Feb 2025 17:06:23 -0800 Subject: [PATCH] Add support for running benchmarks in Linux environment. (#10782) * Add support for running benchmarks in Linux environment. * Remove verbose output for publish benchmark app step * Using template if else instead of condition on task. Using "Invoke-WebRequest" for both windows and Linux. * Fixed displayName to be consistent. * printing the error message conditionally. * remove `condition` for install libssl1.1 step --- eng/ci/host.benchmarks.yml | 34 +++++-- .../official/jobs/run-benchmarks.yml | 99 +++++++++---------- .../official/jobs/setup-benchmark-agents.yml | 74 +++++++++----- 3 files changed, 122 insertions(+), 85 deletions(-) diff --git a/eng/ci/host.benchmarks.yml b/eng/ci/host.benchmarks.yml index 0df171bdb5..6bb75b94d4 100644 --- a/eng/ci/host.benchmarks.yml +++ b/eng/ci/host.benchmarks.yml @@ -44,30 +44,46 @@ extends: - stage: Setup displayName: Setup & start agents jobs: + # WINDOWS + - template: /eng/ci/templates/official/jobs/setup-benchmark-agents.yml@self + parameters: + functionAppName: HelloHttpNet9 + - template: /eng/ci/templates/official/jobs/setup-benchmark-agents.yml@self + parameters: + functionAppName: HelloHttpNet9NoProxy + + # LINUX - template: /eng/ci/templates/official/jobs/setup-benchmark-agents.yml@self parameters: functionAppName: HelloHttpNet9 - agentName: HelloHttpNet9_APP + os: Linux - template: /eng/ci/templates/official/jobs/setup-benchmark-agents.yml@self parameters: - agentName: HelloHttpNet9NoProxy_APP - functionAppName: HelloHttpNet9NoProxy + functionAppName: HelloHttpNet9NoProxy + os: Linux - stage: Run displayName: Run Benchmarks dependsOn: [] # Force this stage to run independently and in parallel with other stages. jobs: + # WINDOWS - template: /eng/ci/templates/official/jobs/run-benchmarks.yml@self parameters: description: .NET9 Web Application functionAppName: HelloHttpNet9 # App with ASP.NET Integration - additionalCrankArgs: $(AdditionalCrankArguments) # Pipeline variable to pass additional arguments to crank command. - storeResultsInDatabase: ${{ variables.storeBenchmarkResultsInDatabase }} - agentName: HelloHttpNet9_APP - template: /eng/ci/templates/official/jobs/run-benchmarks.yml@self parameters: description: .NET9 Worker Application functionAppName: HelloHttpNet9NoProxy # App without ASP.NET Integration - additionalCrankArgs: $(AdditionalCrankArguments) - storeResultsInDatabase: ${{ variables.storeBenchmarkResultsInDatabase }} - agentName: HelloHttpNet9NoProxy_APP + + # LINUX + - template: /eng/ci/templates/official/jobs/run-benchmarks.yml@self + parameters: + description: .NET9 Web Application + functionAppName: HelloHttpNet9 + os: Linux + - template: /eng/ci/templates/official/jobs/run-benchmarks.yml@self + parameters: + description: .NET9 Worker Application + functionAppName: HelloHttpNet9NoProxy + os: Linux diff --git a/eng/ci/templates/official/jobs/run-benchmarks.yml b/eng/ci/templates/official/jobs/run-benchmarks.yml index a11f556199..8fe5a756d5 100644 --- a/eng/ci/templates/official/jobs/run-benchmarks.yml +++ b/eng/ci/templates/official/jobs/run-benchmarks.yml @@ -3,23 +3,32 @@ parameters: type: string - name: description type: string -- name: storeResultsInDatabase - type: boolean - default: false -- name: additionalCrankArgs - type: string - default: '' -- name: agentName +- name: os type: string + default: Windows + values: + - Windows + - Linux jobs: -- job: ${{ parameters.functionAppName }} - +- job: ${{ parameters.functionAppName }}_${{ parameters.os }} + + ${{ if eq(parameters.os, 'Linux') }}: + pool: + name: 1es-pool-azfunc-benchmarking-large + image: 1es-ubuntu-22.04-benchmark-runner-vanilla + os: linux + ${{ else }}: + pool: + name: 1es-pool-azfunc-benchmarking-large + image: 1es-windows-2022-benchmark-runner-vanilla + os: windows + variables: - agentName: ${{ parameters.agentName }} + agentId: ${{ parameters.functionAppName }}${{ parameters.os }} runDescription: ${{ parameters.description }} functionApp: ${{ parameters.functionAppName }} - benchmarkArtifactName: benchmark_results_$(functionApp) + benchmarkArtifactName: benchmark_results_$(Agent.OS)_$(functionApp) functionAppOutputPath: $(Build.ArtifactStagingDirectory)/FunctionApps/$(functionApp) benchmarkResultsJsonPath: $(Build.ArtifactStagingDirectory)/BenchmarkResults/$(Build.BuildNumber)_$(functionApp).json functionsWorkerRuntime: 'dotnet-isolated' @@ -27,7 +36,7 @@ jobs: hostLocation: "./../../" baselineBenchmarkResultFilePath: '' baselineBenchmarkResultsDownloadDir: $(Pipeline.Workspace)/BenchmarkBaselineResult - appAgentIpAddress: '' + appAgentHostName: '' templateContext: inputs: @@ -46,7 +55,7 @@ jobs: steps: - task: AzureKeyVault@2 - condition: and(succeeded(), eq('${{ parameters.storeResultsInDatabase }}', true)) + condition: and(succeeded(), eq(variables['storeBenchmarkResultsInDatabase'], true)) inputs: azureSubscription: Azure-Functions-Host-CI-internal KeyVaultName: functions-perf-crank-kv @@ -61,27 +70,13 @@ jobs: - script: dotnet tool install -g Microsoft.Crank.Controller --version "0.2.0-*" displayName: Install Microsoft.Crank.Controller tool - - pwsh: Start-Process powershell -ArgumentList '-NoExit', '-Command', 'crank-agent' - displayName: Start crank-agent - - - task: CopyFiles@2 - displayName: Copy benchmark apps to temp location - inputs: - SourceFolder: '$(Build.SourcesDirectory)/test/Performance/Apps' - Contents: '**/*' - TargetFolder: '$(Build.ArtifactStagingDirectory)/PerformanceTestApps' - CleanTargetFolder: true - - - task: DotNetCoreCLI@2 - displayName: Publish $(functionApp) app - inputs: - command: publish - publishWebProjects: false - zipAfterPublish: false - modifyOutputPath: false - projects: '$(Build.ArtifactStagingDirectory)/PerformanceTestApps/$(functionApp)/HelloHttp.csproj' - arguments: -c Release -o $(functionAppOutputPath) -f net9.0 - workingDirectory: $(Build.ArtifactStagingDirectory)/PerformanceTestApps/$(functionApp) + - ${{ if eq(parameters.os, 'Windows') }}: + - pwsh: Start-Process powershell -ArgumentList '-NoExit', '-Command', 'crank-agent' + displayName: Start crank-agent + - ${{ else }}: + - script: | + nohup crank-agent & + displayName: Start crank-agent - task: AzureCLI@2 displayName: Get Remote Agent IP Address @@ -99,12 +94,12 @@ jobs: --auth-mode login ` --table-name $(BenchmarkAgentInfoTableName) ` --partition-key $(Build.BuildNumber) ` - --row-key $(agentName) ` - --select AgentIPAddress + --row-key $(agentId) ` + --select HostName if ($Entity) { - $AgentIPAddress = ($Entity | ConvertFrom-Json).AgentIPAddress - Write-Host "##vso[task.setvariable variable=appAgentIpAddress]$AgentIPAddress" + $HostName = ($Entity | ConvertFrom-Json).HostName + Write-Host "##vso[task.setvariable variable=appAgentHostName]$HostName" break } else { Write-Host "Entity not found. Retrying in 30 seconds..." @@ -120,25 +115,24 @@ jobs: - pwsh: | $crankArgs = "--config $(configFilePath) --scenario hellohttp --profile win2022 --load.options.reuseBuild true --description `"$(runDescription)`" --command-line-property --no-measurements --json $(benchmarkResultsJsonPath) --property sourceVersion=$(Build.SourceVersion) --property buildNumber=$(Build.BuildNumber) --property buildId=$(Build.BuildId) --property sourceBranch=$(Build.SourceBranch) --variable FunctionsWorkerRuntime=$(functionsWorkerRuntime) --variable HostLocation=$(hostLocation) --variable FunctionAppPath=$(functionAppOutputPath)" - $crankArgs += " --variable CrankAgentAppVm=$(appAgentIpAddress) --variable CrankAgentLoadVm=localhost --variable AspNetUrls=http://$(appAgentIpAddress):5000" - $crankArgs += " ${{ parameters.additionalCrankArgs }}" + $crankArgs += " --variable CrankAgentAppVm=$(appAgentHostName) --variable CrankAgentLoadVm=localhost --variable AspNetUrls=http://$(appAgentHostName):5000" + $crankArgs += " $(AdditionalCrankArguments)" $command = "crank $crankArgs" - if ('${{ parameters.storeResultsInDatabase }}' -eq 'true') { + if ('${{ variables['storeBenchmarkResultsInDatabase'] }}' -eq 'true') { $command += " --table HttpBenchmarks --sql `"$(BenchmarkResultsSqlConnectionString)`"" } Write-Host "Running command: $command" Invoke-Expression $command - displayName: 'Run Benchmark' + displayName: Run Benchmark - # Retrieve function host logs - pwsh: | - $url = "http://$(appAgentIpAddress):5010/jobs/1/output" + $url = "http://$(appAgentHostName):5010/jobs/1/output" Write-Host "Fetching logs from: $url" $response = Invoke-WebRequest -Uri $url -Method GET -UseBasicParsing Write-Host $response.Content - displayName: 'Fetch Function Host Logs' + displayName: Fetch Function Host Logs # Tag the build as a baseline if it originates from the specified branch. # Baseline builds serve as reference points for performance comparisons in future builds. @@ -152,13 +146,18 @@ jobs: - pwsh: | $baselineDir = "$(baselineBenchmarkResultsDownloadDir)" $fileNamePattern = "*_$(functionApp).json" - $baselineFile = Get-ChildItem -Path $baselineDir -Filter $fileNamePattern | Select-Object -First 1 - if ($baselineFile) { - Write-Host "Found baseline benchmark result file: $($baselineFile.FullName)" - Write-Host "##vso[task.setvariable variable=baselineBenchmarkResultFilePath]$($baselineFile.FullName)" + if (Test-Path $baselineDir -PathType Container) { + $baselineFile = Get-ChildItem -Path $baselineDir -Filter $fileNamePattern | Select-Object -First 1 + + if ($baselineFile) { + Write-Host "Found baseline benchmark result file: $($baselineFile.FullName)" + Write-Host "##vso[task.setvariable variable=baselineBenchmarkResultFilePath]$($baselineFile.FullName)" + } else { + Write-Host "No baseline benchmark result file matching the pattern '$fileNamePattern' found in directory '$baselineDir'." + } } else { - Write-Host "No baseline benchmark result file found." + Write-Host "The specified directory '$baselineDir' does not exist." } displayName: 'Set Baseline Benchmark Result File Path' diff --git a/eng/ci/templates/official/jobs/setup-benchmark-agents.yml b/eng/ci/templates/official/jobs/setup-benchmark-agents.yml index 7c14a1a2c1..8a70d3db04 100644 --- a/eng/ci/templates/official/jobs/setup-benchmark-agents.yml +++ b/eng/ci/templates/official/jobs/setup-benchmark-agents.yml @@ -1,13 +1,29 @@ parameters: - name: functionAppName type: string -- name: agentName +- name: os type: string + default: Windows + values: + - Windows + - Linux + jobs: -- job: ${{ parameters.functionAppName }} +- job: ${{ parameters.functionAppName }}_${{ parameters.os }} + + ${{ if eq(parameters.os, 'Linux') }}: + pool: + name: 1es-pool-azfunc-benchmarking-large + image: 1es-ubuntu-22.04-benchmark-runner-vanilla + os: linux + ${{ else }}: + pool: + name: 1es-pool-azfunc-benchmarking-large + image: 1es-windows-2022-benchmark-runner-vanilla + os: windows variables: - agentName: ${{ parameters.agentName }} + agentId: ${{ parameters.functionAppName }}${{ parameters.os }} functionApp: ${{ parameters.functionAppName }} functionAppOutputPath: $(Build.ArtifactStagingDirectory)/FunctionApps/$(functionApp) @@ -15,15 +31,23 @@ jobs: - template: /eng/ci/templates/install-dotnet.yml@self + - ${{ if eq(parameters.os, 'Linux') }}: + # Adds the Ubuntu 20.04 security repository to Ubuntu 22.04 and installs libssl1.1, which is unavailable by default. + - script: | + echo "deb http://security.ubuntu.com/ubuntu focal-security main" | sudo tee /etc/apt/sources.list.d/focal-security.list + sudo apt update && sudo apt install -y libssl1.1 + displayName: "Install libssl1.1 on Ubuntu 22.04" + - script: dotnet tool install -g Microsoft.Crank.Agent --version "0.2.0-*" displayName: Install Microsoft.Crank.Agent tool - - pwsh: Start-Process powershell -ArgumentList '-NoExit', '-Command', 'crank-agent' - displayName: Start crank agent - - - pwsh: | - dotnet --info - displayName: Print .NET info + - ${{ if eq(parameters.os, 'Windows') }}: + - pwsh: Start-Process powershell -ArgumentList '-NoExit', '-Command', 'crank-agent' + displayName: Start crank-agent + - ${{ else }}: + - script: | + nohup crank-agent & + displayName: Start crank-agent - task: CopyFiles@2 displayName: Copy benchmark apps to temp location @@ -41,22 +65,20 @@ jobs: zipAfterPublish: false modifyOutputPath: false projects: '$(Build.ArtifactStagingDirectory)/PerformanceTestApps/$(functionApp)/HelloHttp.csproj' - arguments: -c Release -o $(functionAppOutputPath) -f net9.0 -v n + arguments: -c Release -o $(functionAppOutputPath) -f net9.0 workingDirectory: $(Build.ArtifactStagingDirectory)/PerformanceTestApps/$(functionApp) - - script: | - netsh advfirewall firewall add rule name="Open Port 5000" dir=in action=allow protocol=TCP localport=5000 - netsh advfirewall firewall add rule name="Open Port 5010" dir=in action=allow protocol=TCP localport=5010 - displayName: Open port 5000 and 5010 + - ${{ if eq(parameters.os, 'Windows') }}: + - script: | + netsh advfirewall firewall add rule name="Open Port 5000" dir=in action=allow protocol=TCP localport=5000 + netsh advfirewall firewall add rule name="Open Port 5010" dir=in action=allow protocol=TCP localport=5010 + displayName: Open port 5000 and 5010 - - task: PowerShell@2 - displayName: Get Agent IP Address - inputs: - targetType: 'inline' - script: | - $ipAddress = (Test-Connection -ComputerName (hostname) -Count 1).IPv4Address.IPAddressToString - Write-Host "IP Address: $ipAddress" - Write-Host "##vso[task.setvariable variable=agentIp]$ipAddress" + - pwsh: | + $HostName = [System.Net.Dns]::GetHostName() + Write-Host "##vso[task.setvariable variable=machineHostName]$HostName" + Write-Host "HostName: $HostName" + displayName: Get Machine Info - task: AzureCLI@2 displayName: Persist Agent IP Address @@ -69,7 +91,7 @@ jobs: --auth-mode login ` --account-name $(StorageAccount) ` --table-name $(BenchmarkAgentInfoTableName) ` - --entity PartitionKey=$(Build.BuildNumber) RowKey=$(agentName) AgentName=$(agentName) AgentIPAddress=$(agentIp) + --entity PartitionKey=$(Build.BuildNumber) RowKey=$(agentId) AgentId=$(agentId) HostName=$(machineHostName) - pwsh: | $url = "http://localhost:5010/jobs/all" @@ -80,9 +102,9 @@ jobs: while ($attempt -lt $maxAttempts) { $response = Invoke-WebRequest -Uri $url -Method Get -ErrorAction Stop $data = $response.Content | ConvertFrom-Json - $completedJobCount = ($data | Where-Object { $_.state -eq "Deleted" }).Count + $completedJobCount = ($data | Where-Object { $_.state -eq "Deleted" }).Count - if ($completedJobCount -gt 0) { + if ($completedJobCount -gt 0) { Write-Host "Found at least 1 job with 'state' = 'Deleted'. Exiting task." exit 0 } @@ -108,4 +130,4 @@ jobs: --auth-mode login ` --account-name $(StorageAccount) ` --table-name $(BenchmarkAgentInfoTableName) ` - --partition-key $(Build.BuildNumber) --row-key=$(agentName) + --partition-key $(Build.BuildNumber) --row-key=$(agentId)