From 1370760cb7ddfdd52c41b1b9f3b6dd1f955d3732 Mon Sep 17 00:00:00 2001 From: Paulo Canilho Date: Sun, 27 Aug 2023 09:05:20 +0100 Subject: [PATCH] release(1.0.4): feat: added API batching support feat: added GitHub Enterprise support feat: added support to delete refs in the format of a branch or tag name feat: added full pagination support feat: add simple concurrency deletion/closing calls; errors will be aggregated and only displayed at the end feat: added test suite for the MiTM API that interacts with GitHub --- .github/workflows/ci.yml | 11 ++ .github/workflows/release.yml | 3 + README.md | 35 ++-- api/api.go | 110 +++++++++++-- api/api_test.go | 292 ++++++++++++++++++++++++++++++++++ api/models.go | 20 +-- cmd/ref.go | 2 +- cmd/root.go | 22 ++- cmd/stale_prs.go | 2 +- delete.txt | 8 - go.mod | 4 + go.sum | 14 ++ helpers/http.go | 57 +++++++ 13 files changed, 526 insertions(+), 54 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 api/api_test.go delete mode 100644 delete.txt create mode 100644 helpers/http.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..8bed778 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,11 @@ +name: ci +on: + push: + branches: [ '**' ] +jobs: + tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - run: | + go test ./... -test.v diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dfda64d..efa2b36 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,8 +7,11 @@ permissions: contents: write jobs: + ci: + uses: ./.github/workflows/ci.yml release: runs-on: ubuntu-latest + needs: [ ci ] steps: - uses: actions/checkout@v3 - uses: cli/gh-extension-precompile@v1 diff --git a/README.md b/README.md index ba023f8..761271d 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,40 @@ # GitHub `tidy` 🧹extension -[![Actions Status: Release](https://github.com/pcanilho/gh-tidy/workflows/release/badge.svg)](https://github.com/pcanilho/gh-tidy/actions?query=release) +[![CI: Tests](https://github.com/pcanilho/gh-tidy/workflows/ci/badge.svg)](https://github.com/pcanilho/gh-tidy/actions?query=ci) +[![CD: Release](https://github.com/pcanilho/gh-tidy/workflows/release/badge.svg)](https://github.com/pcanilho/gh-tidy/actions?query=release) The `gh-tidy` project is a tiny & simple extension for the standard `gh` cli that aims at offering tidy/cleanup operations on existing `refs` -(in either `branch` or `PR` format) by providing rules, such as `stale` status based on last commit date for a given branch, PR activity and others. +(in either `branch`, `tag` or `PR` format) by providing rules, such as `stale` status based on HEAD commit date for a given branch, tag, PR activity and others. + +🚀 Supports: +* **Enterprise** and **Public** GitHub API endpoints are supported. +* **Automatic authentication** using the environment variable `GITHUB_TOKEN`. +* **Automatic** GitHub API **limit handling** where requests are restarted after the `X-RateLimit-Reset` timer expires. +* **Automatic** API **batching** to avoid unnecessary collisions with the internal API (_defaults to `20`_). +* **Listing** & **Deletion** of branches with a stale HEAD commit based on time duration. +* **Listing** & **Deletion** of tags with a stale commit based on time duration. +* **Closing** of PRs with a stale branch HEAD commit based on time duration & PR state. ℹ️ This is a utility project that I have been extending when needed on a best-effort basis. Feel free to contribute with a PR -or open an Issue on GitHub. +or open an Issue on GitHub! -📝 TODOs (for lack of time on my side): +📝 TODOs (for lack of time...): * API: - * Automatic rate limiting handling - * Calls batching - * Support GitHub Enterprise -* Branches: - * Detect if branch has already been merged + * Support GitHub APP `pem` direct authentication. +* [stale] Branches: + * Support optional detected if the provided branch has already been merged to the repository default branch. --- -## Using `gh-tidy` -0. Expose a `GITHUB_TOKEN` environment variable with `repo:read` privileges or `repo:write` if you wish to use the `Delete` features. (*) +## Using `gh-tidy` +_...**locally** or through a CI system like **Jenkins**, **GitHub actions** or any other..._ +0. Expose a `GITHUB_TOKEN` environment variable with `repo:read` privileges or `repo:admin` if you wish to use the `delete` features. (*) 1. Install the `gh` cli available [here](https://github.com/cli/cli#installation). 2. Install the extension: ```shell $ gh extension install pcanilho/gh-tidy ``` - or upgrade to `latest` version: + or upgrade to the `latest` version: ```shell $ gh extension upgrade pcanilho/gh-tidy ``` @@ -34,7 +43,7 @@ or open an Issue on GitHub. $ gh tidy --help ``` -\* This can be a `PAT`, a GitHub App installation `access_token` or any other format that allows `api.github.com` access. +\* This can be a `PAT`, a GitHub App installation `access_token` or any other format that allows API access via `Bearer` token. **Note**: Authentication through direct GitHub App PEM is not (yet) supported. ### Usage diff --git a/api/api.go b/api/api.go index e6dc3a9..7c1ba43 100644 --- a/api/api.go +++ b/api/api.go @@ -5,33 +5,95 @@ import ( "errors" "fmt" "github.com/google/go-github/github" + "github.com/pcanilho/gh-tidy/helpers" "github.com/shurcooL/githubv4" - "golang.org/x/oauth2" - "os" + "net/http" "strings" "sync" "time" ) +const _defaultWorkerCount = 20 + type GitHub struct { - clientV3 *github.Client - clientV4 *githubv4.Client + enterpriseEndpoint string + clientV3 *github.Client + clientV4 *githubv4.Client + + httpClient *http.Client + context context.Context + workerCount int +} + +type Option = func(*GitHub) + +func WithHttpClient(httpClient *http.Client) Option { + return func(session *GitHub) { + session.httpClient = httpClient + } +} + +func WithContext(ctx context.Context) Option { + return func(session *GitHub) { + session.context = ctx + } } -func NewSession() (*GitHub, error) { - token := os.Getenv("GITHUB_TOKEN") - if len(strings.TrimSpace(token)) == 0 { - return nil, fmt.Errorf("a GITHUB_TOKEN environment variable needs to be set") +func WithEnterpriseEndpoint(enterpriseEndpoint string) Option { + return func(session *GitHub) { + session.enterpriseEndpoint = enterpriseEndpoint + } +} + +func WithWorkerCount(workerCount int) Option { + return func(session *GitHub) { + session.workerCount = workerCount + } +} + +func NewSession(opts ...Option) (*GitHub, error) { + inst := new(GitHub) + for _, opt := range opts { + opt(inst) + } + + if inst.context == nil { + inst.context = context.Background() + } + + if inst.workerCount == 0 { + inst.workerCount = _defaultWorkerCount } - ctx := context.Background() - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: token}, - ) - tc := oauth2.NewClient(ctx, ts) - return &GitHub{clientV3: github.NewClient(tc), clientV4: githubv4.NewClient(tc)}, nil + if inst.httpClient == nil { + ghRoundTripper, err := helpers.NewGitHubRoundTripper(inst.context) + if err != nil { + return nil, err + } + inst.httpClient = ghRoundTripper.OauthClient + } + if len(inst.enterpriseEndpoint) != 0 { + clientV3, err := github.NewEnterpriseClient(inst.enterpriseEndpoint, inst.enterpriseEndpoint, inst.httpClient) + if err != nil { + return nil, err + } + inst.clientV3 = clientV3 + inst.clientV4 = githubv4.NewEnterpriseClient(inst.enterpriseEndpoint, inst.httpClient) + } else { + inst.clientV3 = github.NewClient(inst.httpClient) + inst.clientV4 = githubv4.NewClient(inst.httpClient) + } + return inst, nil } func (gh *GitHub) ListPRs(ctx context.Context, states []string, owner, repo string) ([]*GitHubPR, error) { + if len(owner) == 0 { + return nil, fmt.Errorf("an owner must be specified") + } + + if len(repo) == 0 { + return nil, fmt.Errorf("a repo must be specified") + } + var query struct { Repository struct { PullRequests struct { @@ -103,6 +165,14 @@ const ( ) func (gh *GitHub) ListRefs(ctx context.Context, owner, repo string, refType RefType) ([]*GitHubRef, error) { + if len(owner) == 0 { + return nil, fmt.Errorf("an owner must be specified") + } + + if len(repo) == 0 { + return nil, fmt.Errorf("a repo must be specified") + } + var query struct { Repository struct { Refs struct { @@ -176,20 +246,25 @@ func (gh *GitHub) DeleteRefs(ctx context.Context, refs ...string) error { } ec := make(chan error, len(refs)) + sem := make(chan struct{}, gh.workerCount) + var wg sync.WaitGroup wg.Add(len(refs)) go func() { wg.Wait() close(ec) + close(sem) }() for _, ref := range refs { + sem <- struct{}{} go func(r string) { reqErr := gh.clientV4.Mutate(ctx, &mutation, githubv4.Input(r), nil) if reqErr != nil { ec <- fmt.Errorf("unable to delete ref: %v. error: %v", r, reqErr) } wg.Done() + <-sem }(ref) } @@ -212,26 +287,31 @@ func (gh *GitHub) ClosePRs(ctx context.Context, ids ...string) error { } ec := make(chan error, len(ids)) + sem := make(chan struct{}, gh.workerCount) + var wg sync.WaitGroup wg.Add(len(ids)) go func() { wg.Wait() close(ec) + close(sem) }() for _, id := range ids { + sem <- struct{}{} go func(identifier string) { reqErr := gh.clientV4.Mutate(ctx, &mutation, githubv4.Input(identifier), nil) if reqErr != nil { ec <- fmt.Errorf("unable to close PR: %v. error: %v", identifier, reqErr) } wg.Done() + <-sem }(id) } var err error for e := range ec { - err = errors.Join(e) + err = errors.Join(err, e) } return err diff --git a/api/api_test.go b/api/api_test.go new file mode 100644 index 0000000..8a3893b --- /dev/null +++ b/api/api_test.go @@ -0,0 +1,292 @@ +package api_test + +import ( + "context" + "fmt" + "github.com/pcanilho/gh-tidy/api" + "github.com/stretchr/testify/assert" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" +) + +func TestNewSession(t *testing.T) { + t.Run("session-w/o-token", func(ti *testing.T) { + s, err := api.NewSession() + + assert.Error(ti, err) + assert.Nil(ti, s) + }) + t.Run("session-w-token", func(ti *testing.T) { + envKey := "GITHUB_TOKEN" + old := os.Getenv(envKey) + assert.NoError(ti, os.Setenv(envKey, "XXX")) + { + s, err := api.NewSession() + + assert.NoError(ti, err) + assert.NotNil(ti, s) + } + assert.NoError(ti, os.Unsetenv(envKey)) + if len(old) != 0 { + assert.NoError(ti, os.Setenv(envKey, old)) + } + }) +} + +func TestGitHub_ListRefs(t *testing.T) { + setup(t) + refType := new(api.RefType) + t0 := "2023-08-29T19:20:49+01:00" + t0p, terr := time.Parse(time.RFC3339, t0) + owner, repo := "x", "y" + assert.NoError(t, terr) + assert.NotZero(t, t0p) + handler(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, http.MethodPost, r.Method) + assert.Equal(t, + readBody(t, r), + fmt.Sprintf(`{"query":"query($after:String$first:Int!$name:String!$owner:String!$refPrefix:String!){repository(owner: $owner, name: $name){refs(first: $first, after: $after, refPrefix: $refPrefix){nodes{id,name,target{... on Commit{committedDate},... on Tag{tagger{date}}}},pageInfo{endCursor,hasNextPage}}}}","variables":{"after":null,"first":100,"name":"%v","owner":"%v","refPrefix":"%v"}}`, repo, owner, *refType)) + writeBody(t, w, fmt.Sprintf(`{"data": {"repository": {"refs": {"nodes": [{"id": "007", "name": "test-ref", "target": {"committedDate": "%v", "tagger": {"date": "%v"}}}]}}}}`, t0, t0)) + }) + { + *refType = api.BranchRefType + t.Run("list-refs-branches-valid-match", func(ti *testing.T) { + brs, err := ghApi.ListRefs(context.Background(), owner, repo, *refType) + assert.NoError(ti, err) + assert.Len(ti, brs, 1) + expected := &api.GitHubRef{ + Id: "007", + Name: "test-ref", + LastCommitDate: &t0p, + TagDate: &t0p, + } + assert.Equal(ti, expected, brs[0]) + }) + t.Run("list-refs-branches-valid-mismatch", func(ti *testing.T) { + brs, err := ghApi.ListRefs(context.Background(), owner, repo, *refType) + assert.NoError(ti, err) + assert.Len(ti, brs, 1) + expected := &api.GitHubRef{ + Id: "006", + Name: "test-ref", + LastCommitDate: &t0p, + TagDate: &t0p, + } + assert.NotEqual(ti, expected, brs[0]) + }) + *refType = api.TagRefType + t.Run("list-refs-tags-valid-match", func(ti *testing.T) { + brs, err := ghApi.ListRefs(context.Background(), owner, repo, *refType) + assert.NoError(ti, err) + assert.Len(ti, brs, 1) + expected := &api.GitHubRef{ + Id: "007", + Name: "test-ref", + LastCommitDate: &t0p, + TagDate: &t0p, + } + assert.Equal(ti, expected, brs[0]) + }) + t.Run("list-refs-invalid-owner", func(ti *testing.T) { + owner = "" + brs, err := ghApi.ListRefs(context.Background(), owner, repo, *refType) + assert.Error(ti, err) + assert.Nil(ti, brs) + }) + t.Run("list-refs-invalid-repo", func(ti *testing.T) { + repo = "" + owner = "x" + brs, err := ghApi.ListRefs(context.Background(), owner, repo, *refType) + assert.Error(ti, err) + assert.Nil(ti, brs) + }) + } +} + +func TestGitHub_ListPRs(t *testing.T) { + setup(t) + t0 := "2023-08-29T19:20:49+01:00" + t0p, terr := time.Parse(time.RFC3339, t0) + owner, repo := "x", "y" + headName, baseName, url := "test-head-name", "test-base-name", "test-url" + assert.NoError(t, terr) + assert.NotZero(t, t0p) + handler(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, http.MethodPost, r.Method) + assert.Equal(t, + readBody(t, r), + fmt.Sprintf(`{"query":"query($after:String$first:Int!$name:String!$owner:String!$states:[PullRequestState!]!){repository(owner: $owner, name: $name){pullRequests(first: $first, after: $after, states: $states){nodes{id,commits(last: 1){nodes{commit{committedDate},pullRequest{number,url}}},baseRefName,headRefName},pageInfo{endCursor,hasNextPage}}}}","variables":{"after":null,"first":100,"name":"%v","owner":"%v","states":["OPEN"]}}`, repo, owner)) + writeBody(t, w, fmt.Sprintf(`{"data":{"repository":{"pullRequests":{"nodes":[{"id":"007","commits":{"nodes":[{"commit":{"committedDate":"%v"},"pullRequest":{"number":7,"url":"%v"}}]},"baseRefName":"%v","headRefName":"%v"}]}}}}`, t0, url, baseName, headName)) + }) + { + t.Run("list-prs-valid-match", func(ti *testing.T) { + prs, err := ghApi.ListPRs(context.Background(), []string{"OPEN"}, owner, repo) + assert.NoError(ti, err) + assert.Len(ti, prs, 1) + + expected := &api.GitHubPR{ + Id: "007", + Source: headName, + Target: baseName, + LastCommitDate: t0p, + Number: 7, + Url: url, + } + assert.Equal(ti, expected, prs[0]) + }) + t.Run("list-prs-valid-mismatch", func(ti *testing.T) { + prs, err := ghApi.ListPRs(context.Background(), []string{"OPEN"}, owner, repo) + assert.NoError(ti, err) + assert.Len(ti, prs, 1) + + expected := &api.GitHubPR{ + Id: "006", + Source: headName, + Target: baseName, + LastCommitDate: t0p, + Number: 7, + Url: url, + } + assert.NotEqual(ti, expected, prs[0]) + }) + t.Run("list-prs-invalid-owner", func(ti *testing.T) { + owner = "" + prs, err := ghApi.ListPRs(context.Background(), []string{"OPEN"}, owner, repo) + assert.Error(ti, err) + assert.Nil(ti, prs) + }) + t.Run("list-prs-invalid-repo", func(ti *testing.T) { + repo = "" + owner = "x" + prs, err := ghApi.ListPRs(context.Background(), []string{"OPEN"}, owner, repo) + assert.Error(ti, err) + assert.Nil(ti, prs) + }) + } +} + +func TestGitHub_DeleteRefs(t *testing.T) { + setup(t) + ref := "x" + handler(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, http.MethodPost, r.Method) + assert.Equal(t, + readBody(t, r), + fmt.Sprintf(`{"query":"mutation($input:ID!){deleteRef(input: {refId: $input}){typename :__typename}}","variables":{"input":"%v"}}`, ref)) + writeBody(t, w, `{"data":{}}`) + }) + { + t.Run("delete-refs-valid", func(ti *testing.T) { + assert.NoError(ti, + ghApi.DeleteRefs(context.Background(), ref)) + }) + t.Run("delete-refs-invalid-empty", func(ti *testing.T) { + assert.Error(ti, + ghApi.DeleteRefs(context.Background())) + }) + } + setup(t) + handler(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, http.MethodPost, r.Method) + w.WriteHeader(http.StatusInternalServerError) + }) + { + t.Run("delete-refs-errors", func(ti *testing.T) { + err := ghApi.DeleteRefs(context.Background(), "x", "y", "z", "w") + assert.Error(ti, err) + assert.ErrorContains(ti, err, "unable to delete ref: x") + assert.ErrorContains(ti, err, "unable to delete ref: y") + assert.ErrorContains(ti, err, "unable to delete ref: z") + assert.ErrorContains(ti, err, "unable to delete ref: w") + }) + } +} + +func TestGitHub_ClosePRs(t *testing.T) { + setup(t) + identifier := "x" + handler(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, http.MethodPost, r.Method) + assert.Equal(t, + readBody(t, r), + fmt.Sprintf(`{"query":"mutation($input:ID!){closePullRequest(input: {pullRequestId: $input}){typename :__typename}}","variables":{"input":"%v"}}`, identifier)) + writeBody(t, w, `{"data":{}}`) + }) + { + t.Run("close-prs-valid", func(ti *testing.T) { + assert.NoError(ti, + ghApi.ClosePRs(context.Background(), identifier)) + }) + t.Run("close-prs-invalid-empty", func(ti *testing.T) { + assert.Error(ti, + ghApi.ClosePRs(context.Background())) + }) + } + setup(t) + handler(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, http.MethodPost, r.Method) + w.WriteHeader(http.StatusInternalServerError) + }) + { + t.Run("close-prs-errors", func(ti *testing.T) { + err := ghApi.ClosePRs(context.Background(), "x", "y", "z", "w") + assert.Error(ti, err) + assert.ErrorContains(ti, err, "unable to close PR: x") + assert.ErrorContains(ti, err, "unable to close PR: y") + assert.ErrorContains(ti, err, "unable to close PR: z") + assert.ErrorContains(ti, err, "unable to close PR: w") + }) + } +} + +/********************************/ + +var ( + mux *http.ServeMux + ghApi *api.GitHub +) + +func setup(t *testing.T) { + mux = http.NewServeMux() + inst, err := api.NewSession( + api.WithContext(context.Background()), + api.WithHttpClient(&http.Client{Transport: &httpTestServer{handler: mux}})) + assert.NoError(t, err) + assert.NotNil(t, inst) + ghApi = inst +} + +func handler(fn http.HandlerFunc) { + mux.HandleFunc("/graphql", fn) +} + +type httpTestServer struct { + handler http.Handler +} + +func (l *httpTestServer) RoundTrip(req *http.Request) (*http.Response, error) { + w := httptest.NewRecorder() + l.handler.ServeHTTP(w, req) + return w.Result(), nil +} + +func readBody(t *testing.T, r *http.Request) string { + assert.NotNil(t, r) + assert.NotNil(t, r.Body) + body, err := io.ReadAll(r.Body) + assert.NoError(t, err) + assert.NotEmpty(t, body) + return strings.TrimSuffix(string(body), "\n") +} + +func writeBody(t *testing.T, w http.ResponseWriter, body string) { + assert.NotNil(t, w) + n, err := io.WriteString(w, body) + assert.NoError(t, err) + assert.NotZero(t, n) +} diff --git a/api/models.go b/api/models.go index d071e6d..adcd679 100644 --- a/api/models.go +++ b/api/models.go @@ -3,17 +3,17 @@ package api import "time" type GitHubRef struct { - Id string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - LastCommitDate *time.Time `json:"last_commit_date"` - TagDate *time.Time `json:"tag_date"` + Id string `json:"id,omitempty" yaml:"id,omitempty"` + Name string `json:"name,omitempty" yaml:"name,omitempty"` + LastCommitDate *time.Time `json:"last_commit_date,omitempty" yaml:"last_commit_date,omitempty"` + TagDate *time.Time `json:"tag_date,omitempty" yaml:"tag_date"` } type GitHubPR struct { - Id string `json:"id,omitempty"` - Source string `json:"source,omitempty"` - Target string `json:"target,omitempty"` - LastCommitDate time.Time `json:"last_commit_date"` - Number int `json:"number,omitempty"` - Url string `json:"url,omitempty"` + Source string `json:"source,omitempty" yaml:"source,omitempty"` + Target string `json:"target,omitempty" yaml:"target,omitempty"` + LastCommitDate time.Time `json:"last_commit_date" yaml:"last_commit_date"` + Id string `json:"id,omitempty" yaml:"id,omitempty"` + Number int `json:"number,omitempty" yaml:"number,omitempty"` + Url string `json:"url,omitempty" yaml:"url,omitempty"` } diff --git a/cmd/ref.go b/cmd/ref.go index d106292..f6e204e 100644 --- a/cmd/ref.go +++ b/cmd/ref.go @@ -61,7 +61,7 @@ var deleteRefCmd = &cobra.Command{ } } if err = ghApi.DeleteRefs(cmd.Context(), toDeleteIds...); err != nil { - return fmt.Errorf("unable to delete [refs=%v]. error: %v%v", refs, err) + return fmt.Errorf("unable to delete [refs=%v]. error: %v", refs, err) } out = fmt.Sprintf("Deleted [refs=%v] with [ids=%v]\n", refs, toDeleteIds) diff --git a/cmd/root.go b/cmd/root.go index f80f8b7..ddf04ea 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -27,10 +27,12 @@ var ( ) var ( - owner string - format string - force bool - timed bool + owner string + format string + force bool + timed bool + workerCount int + enterpriseUrl string ) var ( @@ -38,8 +40,11 @@ var ( ) var rootCmd = &cobra.Command{ - Use: "gh-tidy", + Use: "tidy", Example: `$ direnv allow || read -s GITHUB_TOKEN; export GITHUB_TOKEN +# Omitting the '--rm' flag runs the command in 'dry-run' mode with the exception of 'delete' command + +$ gh tidy stale branches -t 72h $ gh tidy stale branches -t 72h $ gh tidy stale prs -t 72h -s OPEN -s MERGED $ gh tidy stale tags -t 72h @@ -64,7 +69,10 @@ $ gh tidy delete -t 72h --ref --ref