From 1328e94ceb9c2aa68db20cd461897a4cae175dee Mon Sep 17 00:00:00 2001 From: "Stephen Weatherford (MSFT)" Date: Fri, 4 Oct 2024 09:43:06 -0700 Subject: [PATCH 1/6] Scripts for working with dependabot PRs --- scripts/Fix-Dependabot-PR-Dependencies.ps1 | 22 --- scripts/Fix-DependabotPRs.ps1 | 189 +++++++++++++++++++++ scripts/Recreate-DependabotPRs.ps1 | 4 + 3 files changed, 193 insertions(+), 22 deletions(-) delete mode 100644 scripts/Fix-Dependabot-PR-Dependencies.ps1 create mode 100644 scripts/Fix-DependabotPRs.ps1 create mode 100644 scripts/Recreate-DependabotPRs.ps1 diff --git a/scripts/Fix-Dependabot-PR-Dependencies.ps1 b/scripts/Fix-Dependabot-PR-Dependencies.ps1 deleted file mode 100644 index dc507d66fff..00000000000 --- a/scripts/Fix-Dependabot-PR-Dependencies.ps1 +++ /dev/null @@ -1,22 +0,0 @@ -# This script is used to fix the dependabot PR dependencies by triggering the lockfiles-command workflow for each PR and then closing and reopening the PRs to force a rebuild. - -Write-Host "Running lockfiles-command workflow for each PR..." -gh pr list --label dependencies --limit 100 --json title,headRefName --jq '.[] | select(.title | startswith("Bump")) | .headRefName' | foreach-object { gh workflow run lockfiles-command.yml --ref=$_ } - -Write-Host "Waiting for all workflows to complete..." -while ($true) { - $Running = gh run list --workflow=lockfiles-command.yml --json status,headBranch --jq '.[] | select(.status == "in_progress" or .status == "queued") | .headBranch' - if ($Running.count -ne 0) { - Write-Host "Some workflows are still in progress: $($running -join ', ')" - } - else { - Write-Host "All workflows have completed." - break - } - - # Wait for 1 minute before checking again - Start-Sleep -Seconds 60 -} - -Write-Host "Closing and reopening PRs to force a rebuild..." -gh pr list --label dependencies --limit 100 --json title,number --jq '.[] | select(.title | startswith("Bump")) | .number' | foreach-object { gh pr close $_; gh pr reopen $_ -c "Forcing rebuild" } diff --git a/scripts/Fix-DependabotPRs.ps1 b/scripts/Fix-DependabotPRs.ps1 new file mode 100644 index 00000000000..415574f7cc4 --- /dev/null +++ b/scripts/Fix-DependabotPRs.ps1 @@ -0,0 +1,189 @@ +# This script is used to fix the dependabot PR dependencies by triggering the lockfiles-command workflow for each PR and then closing and reopening the PRs to force a rebuild. +# If you don't want specific PRs to be affected, add a prefix to the title (e.g. "Needs manual intervention: "). This script ignores any PRs that don't start directly with "Bump" + +$maxPRs = 100 +$dryRun = $false + +function getPrLink($prNumber) { + return "https://github.com/azure/bicep/pull/$($prNumber)" +} + +function getPrState($prNumber) { + return (gh pr view $prNumber --json state | jq '.state').Trim('"') +} + +function prHasConflicts($prNumber) { + return (gh pr view $prNumber --json mergeable | jq '.mergeable').Trim('"') -eq 'CONFLICTING' +} + +function waitForPrRecreate($prNumber) { + if (-not (prHasConflicts($prNumber))) { + return + } + + Write-Host -NoNewline "PR $(getPrLink($prNumber)) has conflicts. Waiting for it to be recreated..." + + while ($true) { + $lastComment = gh pr view $prNumber --json comments --jq '.comments[-1].body' + if ($lastComment -notlike '*@dependabot recreate*') { + Write-Host "" + write-warning "PR $(getPrLink($prNumber)) last comment: $lastComment" + return + } + + if (prHasConflicts($prNumber)) { + Write-Host -NoNewline "." + } else { + Write-Host "`nPR $(getPrLink($prNumber)) has been recreated." + return + } + + Start-Sleep -Seconds 60 + } + + Write-Host "" +} + +# returns true if the PR should be removed from the list, otherwise false +function processPR { + param ( + [Parameter(Mandatory = $true)] + [int]$prNumber, + + [Parameter(Mandatory = $true)] + [string]$prRef, + + [ref]$prs # Use a reference to modify the original array asdfg + ) + + Write-Host "`n====================== Processing PR $prNumber ======================`n" + + $prState = getPrState($prNumber) + if ($prState -ne 'OPEN') { + Write-Warning "PR $(getPrLink($prNumber)) is not open. Skipping..." + $prStatus[$prNumber] = "Unexpected closed" + return $true + } + + if ($prStatus[$prNumber] -eq "Conflicts") { + Write-Host "Waiting for PR $(getPrLink($prNumber)) with conflicts to be recreated..." + waitForPrRecreate $prNumber + $prStatus[$prNumber] = "Recreated" + + $prState = getPrState($prNumber) + if ($prState -ne 'OPEN') { + Write-Warning "PR $(getPrLink($prNumber)) was closed during @dependabot recreate." + $prStatus[$prNumber] = "Closed after @dependabot recreate" + return $true + } + } + + if (prHasConflicts($prNumber)) { + Write-Host "PR $(getPrLink($prNumber)) has conflicts. Recreating PR and putting at the end of the list." + if (!$dryRun) { + gh pr comment $prNumber --body "@dependabot recreate" + } + $prStatus[$prNumber] = "Conflicts" + return $false + } + + Write-Host "Running lockfiles-command workflow for PR $(getPrLink($prNumber))" + if (!$dryRun) { + gh workflow run lockfiles-command.yml --ref=$prRef + } + + Write-Host "Waiting for the lockfiles-command workflow to complete for PR $(getPrLink($prNumber))" + if (!$dryRun) { + Start-Sleep -Seconds 15 + } + + while ($true) { + $runningOutput = gh run list --workflow=lockfiles-command.yml --json status,headBranch + $running = $runningOutput | ConvertFrom-Json | Where-Object { $_.headBranch -eq $prRef } + + if (!$running) { + write-host "" + Write-Warning "No lockfiles-command workflows found for PR $(getPrLink($prNumber))" + $prStatus[$prNumber] = "No lockfiles-command workflows found" + return $true + } + + $inProgress = $running | Where-Object { $_.status -eq "in_progress" -or $_.status -eq "queued" } + if ($inProgress) { + Write-Host -NoNewline "." + } else { + Write-Host "`nlockfiles-command for $(getPrLink($prNumber)) has completed." + break + } + + Start-Sleep -Seconds 15 + } + + Write-Host "Closing and reopening $(getPrLink($prNumber)) to force a rebuild..." + if (!$dryRun) { + gh pr close $prNumber + gh pr reopen $prNumber -c "Forcing rebuild" + } + + Write-Host "Waiting for the required checks to complete for $(getPrLink($prNumber))" + if (!$dryRun) { + Start-Sleep -Seconds 15 # Wait for the PR to reopen before checking the status + } + + # Use gh checks wait with --fail-fast to exit on the first failure + gh pr checks $prNumber --watch --fail-fast --required + if ($LASTEXITCODE -ne 0) { + Write-Error "Checks for $(getPrLink($prNumber)) have failed." + $prStatus[$prNumber] = "Checks failed" + return $true + } + + # Set to auto-merge + Write-Host "All checks for $(getPrLink($prNumber)) have completed successfully." + gh pr merge $prNumber --squash --auto + Write-Host "PR $(getPrLink($prNumber)) has been set to auto-merge." + $prStatus[$prNumber] = "Auto-merged" + + return $true +} + +function showStatus($prStatus) { + Write-Host "`nStatus:" + foreach ($pr in $prs) { + Write-Host "$(getPrLink($pr.number)): $($prStatus[$pr.number]) ($(getPrState($pr.number)))" + } + Write-Host "`n" +} +Write-Host "Getting list of matching PRs..." +$prsJson = gh pr list --label dependencies --limit $maxPRs --json title,number,headRefName,state,author --jq '.[] | select(.title | startswith("Bump")) | select(.author.login == "app/dependabot")' +$allPrs = $prsJson | ConvertFrom-Json +$prStatus = @{} +$allPrs | ForEach-Object { $prStatus[$_.number] = "Not yet processed" } + +# Loop through each PR one at a time +$prs = $allPrs +write-host "Processing $($prs.Count) PRs...$($prs | ForEach-Object { "`n$(getPrLink($_.number)): $($_.title)" })" + +while ($prs) { + showStatus $prStatus + + $pr = $prs[0] + $prs = $prs[1..$prs.Length] + + $processed = processPR -prNumber $pr.number -prRef $pr.headRefName -prs ([ref]$prs) + if ($processed[-1] -eq $true) { + Write-Host "PR $(getPrLink($pr.number)) has been processed." + } elseif ($processed[-1] -eq $false) { + Write-Host "PR $(getPrLink($pr.number)) is still being processed." + $prs = $prs + $pr # Put at the end of the list + } else { + Write-Error "Unexpected return value from processPR: $processed" + } +} + +Write-Host "All PRs processed." +showStatus $prStatus + +foreach ($pr in $prs) { + Write-Host "$(getPrLink($pr.number)): $(getPrState($pr.number))" +} \ No newline at end of file diff --git a/scripts/Recreate-DependabotPRs.ps1 b/scripts/Recreate-DependabotPRs.ps1 new file mode 100644 index 00000000000..02c5a9b05d0 --- /dev/null +++ b/scripts/Recreate-DependabotPRs.ps1 @@ -0,0 +1,4 @@ +# This script causes all dependabot PRs to recreate themselves +# If you don't want specific PRs to be affected, add a prefix to the title (e.g. "Needs manual intervention: "). This script ignores any PRs that don't start directly with "Bump" + +gh pr list --label dependencies --limit 100 --json number,title,author --jq '.[] | select((.title | startswith("Bump")) and (.author.login == "app/dependabot")) | .number' | foreach-object {gh pr comment --body "@dependabot recreate" $_} From fdce7e9c4377c97ecda35e746aeb1c0202c69a12 Mon Sep 17 00:00:00 2001 From: "Stephen Weatherford (MSFT)" Date: Wed, 11 Dec 2024 15:19:07 -0800 Subject: [PATCH 2/6] work --- .vscode/launch.json | 7 +++++++ scripts/Fix-DependabotPRs.ps1 | 11 ++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 9945f851cbf..18f809ee2f5 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -94,6 +94,13 @@ "type": "coreclr", "request": "attach", "processId": "${command:pickProcess}" + }, + { + "name": "PowerShell Launch Current File", + "type": "PowerShell", + "request": "launch", + "script": "${file}", + "cwd": "${cwd}" } ] } diff --git a/scripts/Fix-DependabotPRs.ps1 b/scripts/Fix-DependabotPRs.ps1 index 415574f7cc4..a753b3f11a1 100644 --- a/scripts/Fix-DependabotPRs.ps1 +++ b/scripts/Fix-DependabotPRs.ps1 @@ -2,7 +2,7 @@ # If you don't want specific PRs to be affected, add a prefix to the title (e.g. "Needs manual intervention: "). This script ignores any PRs that don't start directly with "Bump" $maxPRs = 100 -$dryRun = $false +$dryRun = $true function getPrLink($prNumber) { return "https://github.com/azure/bicep/pull/$($prNumber)" @@ -147,13 +147,14 @@ function processPR { return $true } -function showStatus($prStatus) { +function showStatus($prs, $prStatus) { Write-Host "`nStatus:" foreach ($pr in $prs) { Write-Host "$(getPrLink($pr.number)): $($prStatus[$pr.number]) ($(getPrState($pr.number)))" } Write-Host "`n" } + Write-Host "Getting list of matching PRs..." $prsJson = gh pr list --label dependencies --limit $maxPRs --json title,number,headRefName,state,author --jq '.[] | select(.title | startswith("Bump")) | select(.author.login == "app/dependabot")' $allPrs = $prsJson | ConvertFrom-Json @@ -165,7 +166,7 @@ $prs = $allPrs write-host "Processing $($prs.Count) PRs...$($prs | ForEach-Object { "`n$(getPrLink($_.number)): $($_.title)" })" while ($prs) { - showStatus $prStatus + showStatus $prs $prStatus $pr = $prs[0] $prs = $prs[1..$prs.Length] @@ -182,8 +183,8 @@ while ($prs) { } Write-Host "All PRs processed." -showStatus $prStatus +showStatus $allPrs $prStatus -foreach ($pr in $prs) { +foreach ($pr in $allPrs) { Write-Host "$(getPrLink($pr.number)): $(getPrState($pr.number))" } \ No newline at end of file From bd02f7bce70152a3cca459a72c677c49ee581da0 Mon Sep 17 00:00:00 2001 From: "Stephen Weatherford (MSFT)" Date: Wed, 11 Dec 2024 15:25:52 -0800 Subject: [PATCH 3/6] fix --- scripts/Fix-DependabotPRs.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/Fix-DependabotPRs.ps1 b/scripts/Fix-DependabotPRs.ps1 index a753b3f11a1..bf1f0f3f318 100644 --- a/scripts/Fix-DependabotPRs.ps1 +++ b/scripts/Fix-DependabotPRs.ps1 @@ -2,7 +2,7 @@ # If you don't want specific PRs to be affected, add a prefix to the title (e.g. "Needs manual intervention: "). This script ignores any PRs that don't start directly with "Bump" $maxPRs = 100 -$dryRun = $true +$dryRun = $false function getPrLink($prNumber) { return "https://github.com/azure/bicep/pull/$($prNumber)" From befc6a06f57cbed64b7b345870a0c62b8f8dc3c9 Mon Sep 17 00:00:00 2001 From: "Stephen Weatherford (MSFT)" Date: Thu, 12 Dec 2024 09:09:03 -0800 Subject: [PATCH 4/6] work --- scripts/Fix-DependabotPRs.ps1 | 50 +++++++++++++++++------------------ 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/scripts/Fix-DependabotPRs.ps1 b/scripts/Fix-DependabotPRs.ps1 index bf1f0f3f318..d16033145d5 100644 --- a/scripts/Fix-DependabotPRs.ps1 +++ b/scripts/Fix-DependabotPRs.ps1 @@ -51,9 +51,7 @@ function processPR { [int]$prNumber, [Parameter(Mandatory = $true)] - [string]$prRef, - - [ref]$prs # Use a reference to modify the original array asdfg + [string]$prRef ) Write-Host "`n====================== Processing PR $prNumber ======================`n" @@ -122,7 +120,7 @@ function processPR { Write-Host "Closing and reopening $(getPrLink($prNumber)) to force a rebuild..." if (!$dryRun) { gh pr close $prNumber - gh pr reopen $prNumber -c "Forcing rebuild" + gh pr reopen $prNumber -c "Fix-DependabotPRs.ps1: Forcing rebuild" } Write-Host "Waiting for the required checks to complete for $(getPrLink($prNumber))" @@ -140,51 +138,51 @@ function processPR { # Set to auto-merge Write-Host "All checks for $(getPrLink($prNumber)) have completed successfully." + gh pr comment $prNumber --body "Fix-DependabotPRs.ps1: All checks have passed. Setting to auto-merge." gh pr merge $prNumber --squash --auto Write-Host "PR $(getPrLink($prNumber)) has been set to auto-merge." - $prStatus[$prNumber] = "Auto-merged" + $prStatus[$prNumber] = "Set to auto merge" return $true } -function showStatus($prs, $prStatus) { - Write-Host "`nStatus:" +function showStatus($prs) { foreach ($pr in $prs) { - Write-Host "$(getPrLink($pr.number)): $($prStatus[$pr.number]) ($(getPrState($pr.number)))" + Write-Host "$(getPrLink($pr.number)): $($prStatus[[int]$pr.number]) ($(getPrState($pr.number)))" } - Write-Host "`n" } Write-Host "Getting list of matching PRs..." $prsJson = gh pr list --label dependencies --limit $maxPRs --json title,number,headRefName,state,author --jq '.[] | select(.title | startswith("Bump")) | select(.author.login == "app/dependabot")' $allPrs = $prsJson | ConvertFrom-Json $prStatus = @{} -$allPrs | ForEach-Object { $prStatus[$_.number] = "Not yet processed" } +$allPrs | ForEach-Object { $prStatus[$_.number] = "" } # Loop through each PR one at a time -$prs = $allPrs -write-host "Processing $($prs.Count) PRs...$($prs | ForEach-Object { "`n$(getPrLink($_.number)): $($_.title)" })" +$prsToBeProcessed = $allPrs +write-host "Processing $($prsToBeProcessed.Count) PRs...$($prsToBeProcessed | ForEach-Object { "`n$(getPrLink($_.number)): $($_.title)" })" -while ($prs) { - showStatus $prs $prStatus +while ($prsToBeProcessed) { + Write-Host "`nStatus:" + showStatus $allPrs + Write-Host "Still in queue:" + showStatus $prsToBeProcessed - $pr = $prs[0] - $prs = $prs[1..$prs.Length] + $pr = $prsToBeProcessed[0] + $prNumber = [int]$pr.number - $processed = processPR -prNumber $pr.number -prRef $pr.headRefName -prs ([ref]$prs) + $processed = processPR -prNumber $prNumber -prRef $pr.headRefName if ($processed[-1] -eq $true) { - Write-Host "PR $(getPrLink($pr.number)) has been processed." + Write-Host "PR $(getPrLink($prNumber)) has been processed." + $prsToBeProcessed = $prsToBeProcessed[1..$prsToBeProcessed.Length] } elseif ($processed[-1] -eq $false) { - Write-Host "PR $(getPrLink($pr.number)) is still being processed." - $prs = $prs + $pr # Put at the end of the list + Write-Host "PR $(getPrLink($prNumber)) is still being processed." + $prsToBeProcessed = $prsToBeProcessed[1..$prsToBeProcessed.Length] + $pr # Put at the end of the list + $prStatus[$prNumber] = $prStatus[$prNumber] + " (sent to back of queue)" } else { Write-Error "Unexpected return value from processPR: $processed" } } -Write-Host "All PRs processed." -showStatus $allPrs $prStatus - -foreach ($pr in $allPrs) { - Write-Host "$(getPrLink($pr.number)): $(getPrState($pr.number))" -} \ No newline at end of file +Write-Host "`nAll PRs processed." +showStatus $allPrs From a7acff074b3b141e154f400535d2f29139d026ac Mon Sep 17 00:00:00 2001 From: "Stephen Weatherford (MSFT)" Date: Thu, 12 Dec 2024 10:08:14 -0800 Subject: [PATCH 5/6] work --- scripts/Fix-DependabotPRs.ps1 | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/Fix-DependabotPRs.ps1 b/scripts/Fix-DependabotPRs.ps1 index d16033145d5..65ca74637a9 100644 --- a/scripts/Fix-DependabotPRs.ps1 +++ b/scripts/Fix-DependabotPRs.ps1 @@ -131,8 +131,12 @@ function processPR { # Use gh checks wait with --fail-fast to exit on the first failure gh pr checks $prNumber --watch --fail-fast --required if ($LASTEXITCODE -ne 0) { - Write-Error "Checks for $(getPrLink($prNumber)) have failed." - $prStatus[$prNumber] = "Checks failed" + $failedChecks = gh pr checks $prNumber --required --json name,state --jq '.[] | select(.state == "FAILURE") | .name' + $failedChecksArray = $failedChecks -split "`n" + $failedChecksCount = $failedChecksArray.Count + $failedChecksString = "$failedChecksCount failed checks: $($failedChecksArray -join ", ")" + Write-Warning $failedChecksString + $prStatus[$prNumber] = $failedChecksString return $true } From a1a1a117c12451296b668eb2d5e10658b491082a Mon Sep 17 00:00:00 2001 From: "Stephen Weatherford (MSFT)" Date: Tue, 7 Jan 2025 10:07:38 -0800 Subject: [PATCH 6/6] #s --- scripts/Fix-DependabotPRs.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/Fix-DependabotPRs.ps1 b/scripts/Fix-DependabotPRs.ps1 index 65ca74637a9..ed9545ca0bc 100644 --- a/scripts/Fix-DependabotPRs.ps1 +++ b/scripts/Fix-DependabotPRs.ps1 @@ -151,8 +151,10 @@ function processPR { } function showStatus($prs) { + $i = 1 foreach ($pr in $prs) { - Write-Host "$(getPrLink($pr.number)): $($prStatus[[int]$pr.number]) ($(getPrState($pr.number)))" + Write-Host "$($i): $(getPrLink($pr.number)): $($prStatus[[int]$pr.number]) ($(getPrState($pr.number)))" + $i++ } } @@ -189,4 +191,5 @@ while ($prsToBeProcessed) { } Write-Host "`nAll PRs processed." +Write-Host "Originally there were $($allPrs.Count) PRs. $($prStatus.Count) are still open." showStatus $allPrs