diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 56ad3904bd6..21871804950 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -4,25 +4,25 @@ - + https://github.com/dotnet/arcade - 2de3acc671fc624191672a45564f9ef130af5cd4 + 8eb29ba860a3cfcfe68f9a8256caa7efc1f1aaba - + https://github.com/dotnet/arcade - 2de3acc671fc624191672a45564f9ef130af5cd4 + 8eb29ba860a3cfcfe68f9a8256caa7efc1f1aaba - + https://github.com/dotnet/arcade - 2de3acc671fc624191672a45564f9ef130af5cd4 + 8eb29ba860a3cfcfe68f9a8256caa7efc1f1aaba - + https://github.com/dotnet/arcade - 2de3acc671fc624191672a45564f9ef130af5cd4 + 8eb29ba860a3cfcfe68f9a8256caa7efc1f1aaba - + https://github.com/dotnet/arcade - 2de3acc671fc624191672a45564f9ef130af5cd4 + 8eb29ba860a3cfcfe68f9a8256caa7efc1f1aaba diff --git a/eng/Versions.props b/eng/Versions.props index 91b59a2a1df..2ba65a5ad34 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -27,11 +27,11 @@ - 1.0.0-beta.19425.1 - 2.0.0-beta.19425.1 - 1.0.0-beta.19425.1 - 2.4.1-beta.19425.1 - 1.0.0-beta.19425.1 + 1.0.0-beta.19461.7 + 2.0.0-beta.19461.7 + 1.0.0-beta.19461.7 + 2.4.1-beta.19461.7 + 1.0.0-beta.19461.7 diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index feb58d14191..e001ccb481c 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -85,6 +85,10 @@ function Build { # Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons. # Explicitly set the type as string[] because otherwise PowerShell would make this char[] if $properties is empty. [string[]] $msbuildArgs = $properties + + # Resolve relative project paths into full paths + $projects = ($projects.Split(';').ForEach({Resolve-Path $_}) -join ';') + $msbuildArgs += "/p:Projects=$projects" $properties = $msbuildArgs } diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index 8854d979f37..46d175fdfdc 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -1,9 +1,9 @@ param ( $darcVersion = $null, - $versionEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16" + $versionEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16", + $verbosity = "m" ) -$verbosity = "m" . $PSScriptRoot\tools.ps1 function InstallDarcCli ($darcVersion) { diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index abdd0bc05a1..242429bca65 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -3,6 +3,7 @@ source="${BASH_SOURCE[0]}" darcVersion='' versionEndpoint="https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16" +verbosity=m while [[ $# > 0 ]]; do opt="$(echo "$1" | awk '{print tolower($0)}')" @@ -15,6 +16,10 @@ while [[ $# > 0 ]]; do versionEndpoint=$2 shift ;; + --verbosity) + verbosity=$2 + shift + ;; *) echo "Invalid argument: $1" usage @@ -34,7 +39,6 @@ while [[ -h "$source" ]]; do [[ $source != /* ]] && source="$scriptroot/$source" done scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" -verbosity=m . "$scriptroot/tools.sh" diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1 index 2a08d5246e7..41416862d91 100644 --- a/eng/common/native/CommonLibrary.psm1 +++ b/eng/common/native/CommonLibrary.psm1 @@ -152,6 +152,8 @@ function Get-File { } else { Write-Verbose "Downloading $Uri" + # Don't display the console progress UI - it's a huge perf hit + $ProgressPreference = 'SilentlyContinue' while($Attempt -Lt $DownloadRetries) { try { diff --git a/eng/common/performance/perfhelixpublish.proj b/eng/common/performance/perfhelixpublish.proj index d07b1fa6e2b..e5826b53237 100644 --- a/eng/common/performance/perfhelixpublish.proj +++ b/eng/common/performance/perfhelixpublish.proj @@ -83,8 +83,8 @@ $(WorkItemDirectory) - $(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --artifacts $(BaselineArtifactsDirectory) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" - $(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" + $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" + $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults);$(FinalCommand) 4:00 @@ -93,8 +93,8 @@ $(WorkItemDirectory) - $(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --artifacts $(ArtifactsDirectory)" - $(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory)" + $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument)" + $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument)" $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults) 4:00 diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index 268986246e5..ec41965fc89 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -17,7 +17,7 @@ Param( [string] $Configurations="CompilationMode=$CompilationMode" ) -$RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") +$RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") -or ($Repository -eq "dotnet-performance") $UseCoreRun = ($CoreRootDirectory -ne [string]::Empty) $UseBaselineCoreRun = ($BaselineCoreRootDirectory -ne [string]::Empty) diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 550b3ebf18e..2f2092166e4 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -113,7 +113,7 @@ while (($# > 0)); do esac done -if [[ "$repository" == "dotnet/performance" ]]; then +if [ "$repository" == "dotnet/performance" ] || [ "$repository" == "dotnet-performance" ]; then run_from_perf_repo=true fi diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index d5ec51b150f..096ac321d12 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -37,10 +37,10 @@ function FirstMatchingSymbolDescriptionOrDefault { # DWARF file for a .dylib $DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf") - $dotnetsymbolExe = "$env:USERPROFILE\.dotnet\tools" - $dotnetsymbolExe = Resolve-Path "$dotnetsymbolExe\dotnet-symbol.exe" + $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools" + $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe" - & $dotnetsymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null + & $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null if (Test-Path $PdbPath) { return "PDB" @@ -159,25 +159,25 @@ function CheckSymbolsAvailable { } } -function Installdotnetsymbol { - $dotnetsymbolPackageName = "dotnet-symbol" +function InstallDotnetSymbol { + $dotnetSymbolPackageName = "dotnet-symbol" $dotnetRoot = InitializeDotNetCli -install:$true $dotnet = "$dotnetRoot\dotnet.exe" $toolList = & "$dotnet" tool list --global - if (($toolList -like "*$dotnetsymbolPackageName*") -and ($toolList -like "*$dotnetsymbolVersion*")) { - Write-Host "dotnet-symbol version $dotnetsymbolVersion is already installed." + if (($toolList -like "*$dotnetSymbolPackageName*") -and ($toolList -like "*$dotnetSymbolVersion*")) { + Write-Host "dotnet-symbol version $dotnetSymbolVersion is already installed." } else { - Write-Host "Installing dotnet-symbol version $dotnetsymbolVersion..." + Write-Host "Installing dotnet-symbol version $dotnetSymbolVersion..." Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." - & "$dotnet" tool install $dotnetsymbolPackageName --version $dotnetsymbolVersion --verbosity "minimal" --global + & "$dotnet" tool install $dotnetSymbolPackageName --version $dotnetSymbolVersion --verbosity "minimal" --global } } try { - Installdotnetsymbol + InstallDotnetSymbol CheckSymbolsAvailable } diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index 6d9bdcf72b0..01799d63ff3 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -45,6 +45,7 @@ if ($GuardianPackageName) { $guardianCliLocation = $GuardianCliLocation } +$workingDirectory = (Split-Path $SourceDirectory -Parent) $ValidPath = Test-Path $guardianCliLocation if ($ValidPath -eq $False) @@ -53,13 +54,13 @@ if ($ValidPath -eq $False) exit 1 } -& $(Join-Path $PSScriptRoot "init-sdl.ps1") -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $ArtifactsDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel -$gdnFolder = Join-Path $ArtifactsDirectory ".gdn" +& $(Join-Path $PSScriptRoot "init-sdl.ps1") -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel +$gdnFolder = Join-Path $workingDirectory ".gdn" if ($TsaOnboard) { if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) { - Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel" - & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel + Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel" + & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel if ($LASTEXITCODE -ne 0) { Write-Host "Guardian tsa-onboard failed with exit code $LASTEXITCODE." exit $LASTEXITCODE @@ -71,10 +72,10 @@ if ($TsaOnboard) { } if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) { - & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams + & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams } if ($SourceToolsList -and $SourceToolsList.Count -gt 0) { - & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams + & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams } if ($UpdateBaseline) { @@ -86,8 +87,8 @@ if ($TsaPublish) { if (-not $TsaRepositoryName) { $TsaRepositoryName = "$($Repository)-$($BranchName)" } - Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel" - & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel + Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel" + & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel if ($LASTEXITCODE -ne 0) { Write-Host "Guardian tsa-publish failed with exit code $LASTEXITCODE." exit $LASTEXITCODE diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 index 26e01c0673c..c737eb0e71c 100644 --- a/eng/common/sdl/init-sdl.ps1 +++ b/eng/common/sdl/init-sdl.ps1 @@ -11,6 +11,9 @@ $ErrorActionPreference = "Stop" Set-StrictMode -Version 2.0 $LASTEXITCODE = 0 +# Don't display the console progress UI - it's a huge perf hit +$ProgressPreference = 'SilentlyContinue' + # Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file $encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken")) $escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn") diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config index 3f97ac2f16c..256ffbfb93a 100644 --- a/eng/common/sdl/packages.config +++ b/eng/common/sdl/packages.config @@ -1,4 +1,4 @@ - + - + diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1 index b90c4399ba1..9bc25314ae2 100644 --- a/eng/common/sdl/run-sdl.ps1 +++ b/eng/common/sdl/run-sdl.ps1 @@ -32,16 +32,16 @@ foreach ($tool in $ToolsList) { Write-Host $tool # We have to manually configure tools that run on source to look at the source directory only if ($tool -eq "credscan") { - Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory : $TargetDirectory `" `" OutputType : pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})" - & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory : $TargetDirectory " "OutputType : pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams}) + Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory < $TargetDirectory `" `" OutputType < pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})" + & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory < $TargetDirectory " "OutputType < pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams}) if ($LASTEXITCODE -ne 0) { Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." exit $LASTEXITCODE } } if ($tool -eq "policheck") { - Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target : $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})" - & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target : $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams}) + Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target < $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})" + & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target < $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams}) if ($LASTEXITCODE -ne 0) { Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." exit $LASTEXITCODE @@ -56,4 +56,4 @@ Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --bas if ($LASTEXITCODE -ne 0) { Write-Host "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE." exit $LASTEXITCODE -} \ No newline at end of file +} diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 91621cf88fd..a7f99641952 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -1,7 +1,10 @@ parameters: overrideParameters: '' # Optional: to override values for parameters. additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' - continueOnError: false # optional: determines whether to continue the build if the step errors; + # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named + # 'continueOnError', the parameter value is not correctly picked up. + # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter + sdlContinueOnError: false # optional: determines whether to continue the build if the step errors; dependsOn: '' # Optional: dependencies of the job jobs: @@ -26,12 +29,12 @@ jobs: -InputPath $(Build.SourcesDirectory)\artifacts\BlobArtifacts -ExtractPath $(Build.SourcesDirectory)\artifacts\BlobArtifacts displayName: Extract Blob Artifacts - continueOnError: ${{ parameters.continueOnError }} + continueOnError: ${{ parameters.sdlContinueOnError }} - powershell: eng/common/sdl/extract-artifact-packages.ps1 -InputPath $(Build.SourcesDirectory)\artifacts\PackageArtifacts -ExtractPath $(Build.SourcesDirectory)\artifacts\PackageArtifacts displayName: Extract Package Artifacts - continueOnError: ${{ parameters.continueOnError }} + continueOnError: ${{ parameters.sdlContinueOnError }} - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' - task: NuGetCommand@2 @@ -45,12 +48,12 @@ jobs: - ${{ if ne(parameters.overrideParameters, '') }}: - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 ${{ parameters.overrideParameters }} displayName: Execute SDL - continueOnError: ${{ parameters.continueOnError }} + continueOnError: ${{ parameters.sdlContinueOnError }} - ${{ if eq(parameters.overrideParameters, '') }}: - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 - -GuardianPackageName Microsoft.Guardian.Cli.0.7.1 + -GuardianPackageName Microsoft.Guardian.Cli.0.7.2 -NugetPackageDirectory $(Build.SourcesDirectory)\.packages -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} displayName: Execute SDL - continueOnError: ${{ parameters.continueOnError }} + continueOnError: ${{ parameters.sdlContinueOnError }} diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 8db456bb7f5..ffda80a197b 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -24,7 +24,7 @@ parameters: workspace: '' -# Job base template specific parameters + # Job base template specific parameters # Optional: Enable installing Microbuild plugin # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix # _TeamName - the name of your team @@ -151,6 +151,9 @@ jobs: continueOnError: ${{ parameters.continueOnError }} condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: NuGetAuthenticate@0 + - ${{ each step in parameters.steps }}: - ${{ step }} diff --git a/eng/common/templates/job/performance.yml b/eng/common/templates/job/performance.yml index ef809253d1a..f877fd7a898 100644 --- a/eng/common/templates/job/performance.yml +++ b/eng/common/templates/job/performance.yml @@ -5,6 +5,7 @@ parameters: displayName: '' # optional -- display name for the job. Will use jobName if not passed pool: '' # required -- name of the Build pool container: '' # required -- name of the container + osGroup: '' # required -- operating system for the job extraSetupParameters: '' # optional -- extra arguments to pass to the setup script frameworks: ['netcoreapp3.0'] # optional -- list of frameworks to run against continueOnError: 'false' # optional -- determines whether to continue the build if the step errors @@ -44,12 +45,13 @@ jobs: - HelixPreCommand: '' - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if eq(variables['Agent.Os'], 'Windows_NT') }}: + - ${{ if eq( parameters.osGroup, 'Windows_NT') }}: - HelixPreCommand: 'set "PERFLAB_UPLOAD_TOKEN=$(PerfCommandUploadToken)"' - IsInternal: -Internal - - ${{ if ne(variables['Agent.Os'], 'Windows_NT') }}: + - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - HelixPreCommand: 'export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"' - IsInternal: --internal + - group: DotNet-HelixApi-Access - group: dotnet-benchview diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 9e77ef1b54b..b722975f9c2 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -47,6 +47,10 @@ jobs: downloadPath: '$(Build.StagingDirectory)/Download' condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: NuGetAuthenticate@0 + - task: PowerShell@2 displayName: Publish Build Assets inputs: @@ -59,6 +63,7 @@ jobs: /p:Configuration=$(_BuildConfig) condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} + - task: powershell@2 displayName: Create ReleaseConfigs Artifact inputs: @@ -67,12 +72,14 @@ jobs: Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) + - task: PublishBuildArtifacts@1 displayName: Publish ReleaseConfigs Artifact inputs: PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt' PublishLocation: Container ArtifactName: ReleaseConfigs + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - task: PublishBuildArtifacts@1 displayName: Publish Logs to VSTS diff --git a/eng/common/templates/post-build/channels/netcore-dev-30.yml b/eng/common/templates/post-build/channels/netcore-dev-30.yml new file mode 100644 index 00000000000..69f1a9013e0 --- /dev/null +++ b/eng/common/templates/post-build/channels/netcore-dev-30.yml @@ -0,0 +1,126 @@ +parameters: + symbolPublishingAdditionalParameters: '' + artifactsPublishingAdditionalParameters: '' + publishInstallersAndChecksums: false + +stages: +- stage: NetCore_Dev30_Publish + dependsOn: validate + variables: + - template: ../common-variables.yml + displayName: .NET Core 3.0 Dev Publishing + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Symbol Publishing + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicDevRelease_30_Channel_Id)) + variables: + - group: DotNet-Symbol-Server-Pats + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + artifactName: 'BlobArtifacts' + continueOnError: true + + - task: DownloadBuildArtifacts@0 + displayName: Download PDB Artifacts + inputs: + artifactName: 'PDBArtifacts' + continueOnError: true + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + /p:Configuration=Release + ${{ parameters.symbolPublishingAdditionalParameters }} + + - job: publish_assets + displayName: Publish Assets + dependsOn: setupMaestroVars + variables: + - group: DotNet-Blob-Feed + - group: AzureDevOps-Artifact-Feeds-Pats + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: IsStableBuild + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicDevRelease_30_Channel_Id)) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests + + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + + # This is necessary whenever we want to publish/restore to an AzDO private feed + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + + - task: PowerShell@2 + displayName: Publish Assets + env: + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-universal-packages-rw) + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + /p:ArtifactsCategory=$(_DotNetArtifactsCategory) + /p:IsStableBuild=$(IsStableBuild) + /p:IsInternalBuild=$(IsInternalBuild) + /p:RepositoryName=$(Build.Repository.Name) + /p:CommitSha=$(Build.SourceVersion) + /p:NugetPath=$(NuGetExeToolPath) + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:Configuration=Release + /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} + /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl) + /p:InstallersAzureAccountKey=$(dotnetcli-storage-key) + /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl) + /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key) + /p:PublishToAzureDevOpsNuGetFeeds=true + /p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3/nuget/v3/index.json' + /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3-transport/nuget/v3/index.json' + /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureDevOpsStaticSymbolsFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3-symbols/nuget/v3/index.json' + /p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + ${{ parameters.artifactsPublishingAdditionalParameters }} + + - template: ../../steps/promote-build.yml + parameters: + ChannelId: ${{ variables.PublicDevRelease_30_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/netcore-dev-31.yml b/eng/common/templates/post-build/channels/netcore-dev-31.yml new file mode 100644 index 00000000000..720a0ab08a2 --- /dev/null +++ b/eng/common/templates/post-build/channels/netcore-dev-31.yml @@ -0,0 +1,126 @@ +parameters: + symbolPublishingAdditionalParameters: '' + artifactsPublishingAdditionalParameters: '' + publishInstallersAndChecksums: false + +stages: +- stage: NetCore_Dev31_Publish + dependsOn: validate + variables: + - template: ../common-variables.yml + displayName: .NET Core 3.1 Dev Publishing + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Symbol Publishing + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicDevRelease_31_Channel_Id)) + variables: + - group: DotNet-Symbol-Server-Pats + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + artifactName: 'BlobArtifacts' + continueOnError: true + + - task: DownloadBuildArtifacts@0 + displayName: Download PDB Artifacts + inputs: + artifactName: 'PDBArtifacts' + continueOnError: true + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + /p:Configuration=Release + ${{ parameters.symbolPublishingAdditionalParameters }} + + - job: publish_assets + displayName: Publish Assets + dependsOn: setupMaestroVars + variables: + - group: DotNet-Blob-Feed + - group: AzureDevOps-Artifact-Feeds-Pats + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: IsStableBuild + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicDevRelease_31_Channel_Id)) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests + + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + + # This is necessary whenever we want to publish/restore to an AzDO private feed + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + + - task: PowerShell@2 + displayName: Publish Assets + env: + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-universal-packages-rw) + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + /p:ArtifactsCategory=$(_DotNetArtifactsCategory) + /p:IsStableBuild=$(IsStableBuild) + /p:IsInternalBuild=$(IsInternalBuild) + /p:RepositoryName=$(Build.Repository.Name) + /p:CommitSha=$(Build.SourceVersion) + /p:NugetPath=$(NuGetExeToolPath) + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:Configuration=Release + /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} + /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl) + /p:InstallersAzureAccountKey=$(dotnetcli-storage-key) + /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl) + /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key) + /p:PublishToAzureDevOpsNuGetFeeds=true + /p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json' + /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json' + /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureDevOpsStaticSymbolsFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json' + /p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + ${{ parameters.artifactsPublishingAdditionalParameters }} + + - template: ../../steps/promote-build.yml + parameters: + ChannelId: ${{ variables.PublicDevRelease_31_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/netcore-dev-5.yml b/eng/common/templates/post-build/channels/netcore-dev-5.yml index 812def31547..9c81e39e9cf 100644 --- a/eng/common/templates/post-build/channels/netcore-dev-5.yml +++ b/eng/common/templates/post-build/channels/netcore-dev-5.yml @@ -1,5 +1,4 @@ parameters: - enableSymbolValidation: true symbolPublishingAdditionalParameters: '' artifactsPublishingAdditionalParameters: '' publishInstallersAndChecksums: false @@ -9,7 +8,7 @@ stages: dependsOn: validate variables: - template: ../common-variables.yml - displayName: .NET Core 5 Dev Channel + displayName: .NET Core 5 Dev Publishing jobs: - template: ../setup-maestro-vars.yml @@ -39,15 +38,15 @@ stages: inputs: filePath: eng\common\sdk-task.ps1 arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet - /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) - /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' /p:Configuration=Release ${{ parameters.symbolPublishingAdditionalParameters }} - - job: + - job: publish_assets displayName: Publish Assets dependsOn: setupMaestroVars variables: @@ -79,87 +78,49 @@ stages: buildType: current artifactName: AssetManifests + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + + # This is necessary whenever we want to publish/restore to an AzDO private feed + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + - task: PowerShell@2 - displayName: Add Assets Location + displayName: Publish Assets env: - AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-universal-packages-rw) inputs: filePath: eng\common\sdk-task.ps1 - arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet - /p:ChannelId=$(NetCore_5_Dev_Channel_Id) + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet /p:ArtifactsCategory=$(_DotNetArtifactsCategory) /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) /p:CommitSha=$(Build.SourceVersion) - /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe - /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' - /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' - /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' - /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:NugetPath=$(NuGetExeToolPath) + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' /p:Configuration=Release + /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl) /p:InstallersAzureAccountKey=$(dotnetcli-storage-key) - /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl) /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key) + /p:PublishToAzureDevOpsNuGetFeeds=true + /p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' + /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' + /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureDevOpsStaticSymbolsFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + /p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' ${{ parameters.artifactsPublishingAdditionalParameters }} - - - task: NuGetCommand@2 - displayName: Publish Packages to AzDO Feed - condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') - inputs: - command: push - vstsFeed: $(AzDoFeedName) - packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg - publishVstsFeed: $(AzDoFeedName) - - - task: PowerShell@2 - displayName: Publish Blobs to AzDO Feed - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 - arguments: -FeedName $(AzDoFeedName) - -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) - enabled: false - - -- stage: NetCore_Dev5_PublishValidation - displayName: Publish Validation - variables: - - template: ../common-variables.yml - jobs: - - template: ../setup-maestro-vars.yml - - - ${{ if eq(parameters.enableSymbolValidation, 'true') }}: - - job: - displayName: Symbol Availability - dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.NetCore_5_Dev_Channel_Id)) - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts - - - task: PowerShell@2 - displayName: Check Symbol Availability - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) - - - template: ../darc-gather-drop.yml - parameters: - ChannelId: ${{ variables.NetCore_5_Dev_Channel_Id }} - - template: ../promote-build.yml - parameters: - ChannelId: ${{ variables.NetCore_5_Dev_Channel_Id }} + - template: ../../steps/promote-build.yml + parameters: + ChannelId: ${{ variables.NetCore_5_Dev_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/netcore-internal-30.yml b/eng/common/templates/post-build/channels/netcore-internal-30.yml new file mode 100644 index 00000000000..594a1a9a78d --- /dev/null +++ b/eng/common/templates/post-build/channels/netcore-internal-30.yml @@ -0,0 +1,124 @@ +parameters: + symbolPublishingAdditionalParameters: '' + artifactsPublishingAdditionalParameters: '' + +stages: +- stage: NetCore_30_Internal_Servicing_Publishing + dependsOn: validate + variables: + - template: ../common-variables.yml + displayName: .NET Core 3.0 Internal Servicing Publishing + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Symbol Publishing + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.InternalServicing_30_Channel_Id)) + variables: + - group: DotNet-Symbol-Server-Pats + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + artifactName: 'BlobArtifacts' + continueOnError: true + + - task: DownloadBuildArtifacts@0 + displayName: Download PDB Artifacts + inputs: + artifactName: 'PDBArtifacts' + continueOnError: true + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + /p:Configuration=Release + ${{ parameters.symbolPublishingAdditionalParameters }} + + - job: publish_assets + displayName: Publish Assets + dependsOn: setupMaestroVars + variables: + - group: DotNet-Blob-Feed + - group: AzureDevOps-Artifact-Feeds-Pats + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: IsStableBuild + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.InternalServicing_30_Channel_Id)) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests + + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + + # This is necessary whenever we want to publish/restore to an AzDO private feed + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + + - task: PowerShell@2 + displayName: Publish Assets + env: + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-universal-packages-rw) + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + /p:IsStableBuild=$(IsStableBuild) + /p:IsInternalBuild=$(IsInternalBuild) + /p:RepositoryName=$(Build.Repository.Name) + /p:CommitSha=$(Build.SourceVersion) + /p:NugetPath=$(NuGetExeToolPath) + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:Configuration=Release + /p:PublishInstallersAndChecksums=true + /p:ChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl) + /p:ChecksumsTargetStaticFeedKey=$(InternalChecksumsBlobFeedKey) + /p:InstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl) + /p:InstallersTargetStaticFeedKey=$(InternalInstallersBlobFeedKey) + /p:PublishToAzureDevOpsNuGetFeeds=true + /p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v3/index.json' + /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v3/index.json' + /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureDevOpsStaticSymbolsFeed='https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-symbols/nuget/v3/index.json' + /p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + ${{ parameters.artifactsPublishingAdditionalParameters }} + + - template: ../../steps/promote-build.yml + parameters: + ChannelId: ${{ variables.InternalServicing_30_Channel_Id }} \ No newline at end of file diff --git a/eng/common/templates/post-build/channels/netcore-release-30.yml b/eng/common/templates/post-build/channels/netcore-release-30.yml new file mode 100644 index 00000000000..8ba9237ffc0 --- /dev/null +++ b/eng/common/templates/post-build/channels/netcore-release-30.yml @@ -0,0 +1,126 @@ +parameters: + symbolPublishingAdditionalParameters: '' + artifactsPublishingAdditionalParameters: '' + publishInstallersAndChecksums: false + +stages: +- stage: NetCore_Release30_Publish + dependsOn: validate + variables: + - template: ../common-variables.yml + displayName: .NET Core 3.0 Release Publishing + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Symbol Publishing + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_30_Channel_Id)) + variables: + - group: DotNet-Symbol-Server-Pats + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + artifactName: 'BlobArtifacts' + continueOnError: true + + - task: DownloadBuildArtifacts@0 + displayName: Download PDB Artifacts + inputs: + artifactName: 'PDBArtifacts' + continueOnError: true + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + /p:Configuration=Release + ${{ parameters.symbolPublishingAdditionalParameters }} + + - job: publish_assets + displayName: Publish Assets + dependsOn: setupMaestroVars + variables: + - group: DotNet-Blob-Feed + - group: AzureDevOps-Artifact-Feeds-Pats + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: IsStableBuild + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_30_Channel_Id)) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests + + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + + # This is necessary whenever we want to publish/restore to an AzDO private feed + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + + - task: PowerShell@2 + displayName: Publish Assets + env: + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-universal-packages-rw) + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + /p:ArtifactsCategory=$(_DotNetArtifactsCategory) + /p:IsStableBuild=$(IsStableBuild) + /p:IsInternalBuild=$(IsInternalBuild) + /p:RepositoryName=$(Build.Repository.Name) + /p:CommitSha=$(Build.SourceVersion) + /p:NugetPath=$(NuGetExeToolPath) + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:Configuration=Release + /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} + /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl) + /p:InstallersAzureAccountKey=$(dotnetcli-storage-key) + /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl) + /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key) + /p:PublishToAzureDevOpsNuGetFeeds=true + /p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3/nuget/v3/index.json' + /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3-transport/nuget/v3/index.json' + /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureDevOpsStaticSymbolsFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3-symbols/nuget/v3/index.json' + /p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + ${{ parameters.artifactsPublishingAdditionalParameters }} + + - template: ../../steps/promote-build.yml + parameters: + ChannelId: ${{ variables.PublicRelease_30_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/netcore-release-31.yml b/eng/common/templates/post-build/channels/netcore-release-31.yml new file mode 100644 index 00000000000..d8270eadaea --- /dev/null +++ b/eng/common/templates/post-build/channels/netcore-release-31.yml @@ -0,0 +1,126 @@ +parameters: + symbolPublishingAdditionalParameters: '' + artifactsPublishingAdditionalParameters: '' + publishInstallersAndChecksums: false + +stages: +- stage: NetCore_Release31_Publish + dependsOn: validate + variables: + - template: ../common-variables.yml + displayName: .NET Core 3.1 Release Publishing + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Symbol Publishing + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_31_Channel_Id)) + variables: + - group: DotNet-Symbol-Server-Pats + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + artifactName: 'BlobArtifacts' + continueOnError: true + + - task: DownloadBuildArtifacts@0 + displayName: Download PDB Artifacts + inputs: + artifactName: 'PDBArtifacts' + continueOnError: true + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + /p:Configuration=Release + ${{ parameters.symbolPublishingAdditionalParameters }} + + - job: publish_assets + displayName: Publish Assets + dependsOn: setupMaestroVars + variables: + - group: DotNet-Blob-Feed + - group: AzureDevOps-Artifact-Feeds-Pats + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: IsStableBuild + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_31_Channel_Id)) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests + + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + + # This is necessary whenever we want to publish/restore to an AzDO private feed + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + + - task: PowerShell@2 + displayName: Publish Assets + env: + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-universal-packages-rw) + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + /p:ArtifactsCategory=$(_DotNetArtifactsCategory) + /p:IsStableBuild=$(IsStableBuild) + /p:IsInternalBuild=$(IsInternalBuild) + /p:RepositoryName=$(Build.Repository.Name) + /p:CommitSha=$(Build.SourceVersion) + /p:NugetPath=$(NuGetExeToolPath) + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:Configuration=Release + /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} + /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl) + /p:InstallersAzureAccountKey=$(dotnetcli-storage-key) + /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl) + /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key) + /p:PublishToAzureDevOpsNuGetFeeds=true + /p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json' + /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json' + /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureDevOpsStaticSymbolsFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json' + /p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + ${{ parameters.artifactsPublishingAdditionalParameters }} + + - template: ../../steps/promote-build.yml + parameters: + ChannelId: ${{ variables.PublicRelease_31_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/netcore-tools-latest.yml b/eng/common/templates/post-build/channels/netcore-tools-latest.yml index c2d2076730b..c75d1867339 100644 --- a/eng/common/templates/post-build/channels/netcore-tools-latest.yml +++ b/eng/common/templates/post-build/channels/netcore-tools-latest.yml @@ -1,5 +1,4 @@ parameters: - enableSymbolValidation: true symbolPublishingAdditionalParameters: '' artifactsPublishingAdditionalParameters: '' publishInstallersAndChecksums: false @@ -9,7 +8,7 @@ stages: dependsOn: validate variables: - template: ../common-variables.yml - displayName: .NET Tools - Latest + displayName: .NET Tools - Latest Publishing jobs: - template: ../setup-maestro-vars.yml @@ -39,15 +38,15 @@ stages: inputs: filePath: eng\common\sdk-task.ps1 arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet - /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) - /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' /p:Configuration=Release ${{ parameters.symbolPublishingAdditionalParameters }} - - job: + - job: publish_assets displayName: Publish Assets dependsOn: setupMaestroVars variables: @@ -79,87 +78,49 @@ stages: buildType: current artifactName: AssetManifests + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + + # This is necessary whenever we want to publish/restore to an AzDO private feed + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + - task: PowerShell@2 - displayName: Add Assets Location + displayName: Publish Assets env: - AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-universal-packages-rw) inputs: filePath: eng\common\sdk-task.ps1 - arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet - /p:ChannelId=$(NetCore_Tools_Latest_Channel_Id) + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet /p:ArtifactsCategory=$(_DotNetArtifactsCategory) /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) /p:CommitSha=$(Build.SourceVersion) - /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe - /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' - /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' - /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' - /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:NugetPath=$(NuGetExeToolPath) + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' /p:Configuration=Release - /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl) /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} + /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl) /p:InstallersAzureAccountKey=$(dotnetcli-storage-key) /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl) /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key) + /p:PublishToAzureDevOpsNuGetFeeds=true + /p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureDevOpsStaticSymbolsFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' + /p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' ${{ parameters.artifactsPublishingAdditionalParameters }} - - - task: NuGetCommand@2 - displayName: Publish Packages to AzDO Feed - condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') - inputs: - command: push - vstsFeed: $(AzDoFeedName) - packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg - publishVstsFeed: $(AzDoFeedName) - - - task: PowerShell@2 - displayName: Publish Blobs to AzDO Feed - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 - arguments: -FeedName $(AzDoFeedName) - -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) - enabled: false - - -- stage: NetCore_Tools_Latest_PublishValidation - displayName: Publish Validation - variables: - - template: ../common-variables.yml - jobs: - - template: ../setup-maestro-vars.yml - - - ${{ if eq(parameters.enableSymbolValidation, 'true') }}: - - job: - displayName: Symbol Availability - dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.NetCore_Tools_Latest_Channel_Id)) - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts - - - task: PowerShell@2 - displayName: Check Symbol Availability - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) - - - template: ../darc-gather-drop.yml - parameters: - ChannelId: ${{ variables.NetCore_Tools_Latest_Channel_Id }} - - template: ../promote-build.yml - parameters: - ChannelId: ${{ variables.NetCore_Tools_Latest_Channel_Id }} + - template: ../../steps/promote-build.yml + parameters: + ChannelId: ${{ variables.NetCore_Tools_Latest_Channel_Id }} \ No newline at end of file diff --git a/eng/common/templates/post-build/channels/public-validation-release.yml b/eng/common/templates/post-build/channels/public-validation-release.yml index 12124d62157..fb2c23d0f4d 100644 --- a/eng/common/templates/post-build/channels/public-validation-release.yml +++ b/eng/common/templates/post-build/channels/public-validation-release.yml @@ -7,11 +7,11 @@ stages: dependsOn: validate variables: - template: ../common-variables.yml - displayName: Validation Channel + displayName: .NET Tools - Validation Publishing jobs: - template: ../setup-maestro-vars.yml - - job: + - job: publish_assets displayName: Publish Assets dependsOn: setupMaestroVars variables: @@ -43,67 +43,49 @@ stages: buildType: current artifactName: AssetManifests + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + + # This is necessary whenever we want to publish/restore to an AzDO private feed + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + - task: PowerShell@2 - displayName: Add Assets Location + displayName: Publish Assets env: - AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-universal-packages-rw) inputs: filePath: eng\common\sdk-task.ps1 - arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet - /p:ChannelId=$(PublicValidationRelease_30_Channel_Id) + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet /p:ArtifactsCategory=$(_DotNetValidationArtifactsCategory) /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) /p:CommitSha=$(Build.SourceVersion) - /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe - /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' - /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' - /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' - /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' + /p:NugetPath=$(NuGetExeToolPath) + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' /p:Configuration=Release + /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl) /p:InstallersAzureAccountKey=$(dotnetcli-storage-key) - /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl) /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key) + /p:PublishToAzureDevOpsNuGetFeeds=true + /p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureDevOpsStaticSymbolsFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' + /p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' ${{ parameters.artifactsPublishingAdditionalParameters }} - - task: NuGetCommand@2 - displayName: Publish Packages to AzDO Feed - condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') - inputs: - command: push - vstsFeed: $(AzDoFeedName) - packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg - publishVstsFeed: $(AzDoFeedName) - - - task: PowerShell@2 - displayName: Publish Blobs to AzDO Feed - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 - arguments: -FeedName $(AzDoFeedName) - -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) - enabled: false - - -- stage: PVR_PublishValidation - displayName: Publish Validation - variables: - - template: ../common-variables.yml - jobs: - - template: ../setup-maestro-vars.yml - - - template: ../darc-gather-drop.yml - parameters: - ChannelId: ${{ variables.PublicValidationRelease_30_Channel_Id }} - - - template: ../promote-build.yml - parameters: - ChannelId: ${{ variables.PublicValidationRelease_30_Channel_Id }} + - template: ../../steps/promote-build.yml + parameters: + ChannelId: ${{ variables.PublicValidationRelease_30_Channel_Id }} diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index b00d85d8cee..adb2a854f20 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -1,11 +1,16 @@ variables: - group: Publish-Build-Assets - group: DotNet-DotNetCli-Storage + - group: DotNet-MSRC-Storage # .NET Core 3 Dev - name: PublicDevRelease_30_Channel_Id value: 3 + # .NET Core 3.1 Dev + - name: PublicDevRelease_31_Channel_Id + value: 128 + # .NET Core 5 Dev - name: NetCore_5_Dev_Channel_Id value: 131 @@ -26,14 +31,14 @@ variables: - name: PublicRelease_30_Channel_Id value: 19 + # .NET Core 3.1 Release + - name: PublicRelease_31_Channel_Id + value: 129 + # Whether the build is internal or not - name: IsInternalBuild value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} - # Storage account name for proxy-backed feeds - - name: ProxyBackedFeedsAccountName - value: dotnetfeed - # Default Maestro++ API Endpoint and API Version - name: MaestroApiEndPoint value: "https://maestro-prod.westus2.cloudapp.azure.com" @@ -47,8 +52,23 @@ variables: - name: SymbolToolVersion value: 1.0.1 + # Feed Configurations + # These should include the suffix "/index.json" + # Default locations for Installers and checksums + # Public Locations - name: ChecksumsBlobFeedUrl - value: https://dotnetcli.blob.core.windows.net/dotnet/index.json - - name: InstallersBlobFeedUrl value: https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json + - name: InstallersBlobFeedUrl + value: https://dotnetcli.blob.core.windows.net/dotnet/index.json + + # Private Locations + - name: InternalChecksumsBlobFeedUrl + value: https://dotnetclichecksumsmsrc.blob.core.windows.net/dotnet/index.json + - name: InternalChecksumsBlobFeedKey + value: $(dotnetclichecksumsmsrc-storage-key) + + - name: InternalInstallersBlobFeedUrl + value: https://dotnetclimsrc.blob.core.windows.net/dotnet/index.json + - name: InternalInstallersBlobFeedKey + value: $(dotnetclimsrc-access-key) diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 3f239fae2bb..5b9d0a5d99b 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -1,11 +1,12 @@ parameters: - enableSourceLinkValidation: true + enableSourceLinkValidation: false enableSigningValidation: true - enableSymbolValidation: true + enableSymbolValidation: false enableNugetValidation: true publishInstallersAndChecksums: false SDLValidationParameters: enable: false + continueOnError: false params: '' # These parameters let the user customize the call to sdk-task.ps1 for publishing @@ -92,24 +93,28 @@ stages: - template: /eng/common/templates/job/execute-sdl.yml parameters: additionalParameters: ${{ parameters.SDLValidationParameters.params }} + continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} - template: \eng\common\templates\post-build\channels\netcore-dev-5.yml parameters: - enableSymbolValidation: ${{ parameters.enableSymbolValidation }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} -- template: \eng\common\templates\post-build\channels\public-dev-release.yml +- template: \eng\common\templates\post-build\channels\netcore-dev-30.yml + parameters: + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + +- template: \eng\common\templates\post-build\channels\netcore-dev-31.yml parameters: - enableSymbolValidation: ${{ parameters.enableSymbolValidation }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - template: \eng\common\templates\post-build\channels\netcore-tools-latest.yml parameters: - enableSymbolValidation: ${{ parameters.enableSymbolValidation }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -119,12 +124,19 @@ stages: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} -- template: \eng\common\templates\post-build\channels\public-release.yml +- template: \eng\common\templates\post-build\channels\netcore-release-30.yml parameters: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + +- template: \eng\common\templates\post-build\channels\netcore-release-31.yml + parameters: + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} -- template: \eng\common\templates\post-build\channels\internal-servicing.yml +- template: \eng\common\templates\post-build\channels\netcore-internal-30.yml parameters: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} diff --git a/eng/common/templates/steps/promote-build.yml b/eng/common/templates/steps/promote-build.yml new file mode 100644 index 00000000000..b90404435dd --- /dev/null +++ b/eng/common/templates/steps/promote-build.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Add Build to Channel + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/promote-build.ps1 + arguments: -BuildId $(BARBuildId) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroApiAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 9c12b1b4fdb..bb5638930ab 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -163,6 +163,7 @@ function GetDotNetInstallScript([string] $dotnetRoot) { $installScript = Join-Path $dotnetRoot "dotnet-install.ps1" if (!(Test-Path $installScript)) { Create-Directory $dotnetRoot + $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit Invoke-WebRequest "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1" -OutFile $installScript } @@ -282,6 +283,7 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { Create-Directory $packageDir Write-Host "Downloading $packageName $packageVersion" + $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit Invoke-WebRequest "https://dotnet.myget.org/F/roslyn-tools/api/v2/package/$packageName/$packageVersion/" -OutFile $packagePath Unzip $packagePath $packageDir } diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 3af9be6157f..94a1edd7d0b 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -208,12 +208,19 @@ function GetDotNetInstallScript { # Use curl if available, otherwise use wget if command -v curl > /dev/null; then - curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" - else - wget -q -O "$install_script" "$install_script_url" + curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { + local exit_code=$? + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." + ExitWithExitCode $exit_code + } + else + wget -q -O "$install_script" "$install_script_url" || { + local exit_code=$? + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." + ExitWithExitCode $exit_code + } fi fi - # return value _GetDotNetInstallScript="$install_script" } diff --git a/global.json b/global.json index 3da6f266aed..bcf877d1399 100644 --- a/global.json +++ b/global.json @@ -11,7 +11,7 @@ "version": "3.0.100-preview6-012264" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19425.1", - "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19425.1" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19461.7", + "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19461.7" } }