Skip to content

Commit

Permalink
Feat bitbucket support (#1890)
Browse files Browse the repository at this point in the history
* bb support
  • Loading branch information
motatoes authored Mar 2, 2025
1 parent bab3696 commit 3f0a92b
Show file tree
Hide file tree
Showing 47 changed files with 1,477 additions and 152 deletions.
15 changes: 11 additions & 4 deletions backend/bootstrap/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,6 @@ package bootstrap
import (
"embed"
"fmt"
"github.com/diggerhq/digger/backend/config"
"github.com/diggerhq/digger/backend/segment"
pprof_gin "github.com/gin-contrib/pprof"
"html/template"
"io/fs"
"log"
Expand All @@ -15,6 +12,10 @@ import (
"runtime"
"runtime/pprof"

"github.com/diggerhq/digger/backend/config"
"github.com/diggerhq/digger/backend/segment"
pprof_gin "github.com/gin-contrib/pprof"

"time"

"github.com/diggerhq/digger/backend/controllers"
Expand Down Expand Up @@ -216,14 +217,20 @@ func Bootstrap(templates embed.FS, diggerController controllers.DiggerController

if enableApi := os.Getenv("DIGGER_ENABLE_API_ENDPOINTS"); enableApi == "true" {
apiGroup := r.Group("/api")
apiGroup.Use(middleware.HeadersApiAuth())
apiGroup.Use(middleware.InternalApiAuth(), middleware.HeadersApiAuth())

reposApiGroup := apiGroup.Group("/repos")
reposApiGroup.GET("/", controllers.ListReposApi)
reposApiGroup.GET("/:repo_id/jobs", controllers.GetJobsForRepoApi)

githubApiGroup := apiGroup.Group("/github")
githubApiGroup.POST("/link", controllers.LinkGithubInstallationToOrgApi)

vcsApiGroup := apiGroup.Group("/connections")
vcsApiGroup.GET("/:id", controllers.GetVCSConnection)
vcsApiGroup.GET("/", controllers.ListVCSConnectionsApi)
vcsApiGroup.POST("/", controllers.CreateVCSConnectionApi)
vcsApiGroup.DELETE("/:id", controllers.DeleteVCSConnection)
}

return r
Expand Down
1 change: 0 additions & 1 deletion backend/controllers/bitbucket.go

This file was deleted.

2 changes: 1 addition & 1 deletion backend/controllers/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func (d DiggerController) UpdateRepoCache(c *gin.Context) {

// update the cache here, do it async for immediate response
go func() {
err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, func(dir string) error {
err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error {
diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml"))
diggerYmlStr = string(diggerYmlBytes)
config, _, _, err = dg_configuration.LoadDiggerConfig(dir, true, nil)
Expand Down
199 changes: 199 additions & 0 deletions backend/controllers/connections.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,199 @@
package controllers

import (
"errors"
"github.com/samber/lo"
"log"
"net/http"
"os"

"github.com/diggerhq/digger/backend/utils"
"gorm.io/gorm"

"github.com/diggerhq/digger/backend/middleware"
"github.com/diggerhq/digger/backend/models"
"github.com/gin-gonic/gin"
)

func ListVCSConnectionsApi(c *gin.Context) {
organisationId := c.GetString(middleware.ORGANISATION_ID_KEY)
organisationSource := c.GetString(middleware.ORGANISATION_SOURCE_KEY)

var org models.Organisation
err := models.DB.GormDB.Where("external_id = ? AND external_source = ?", organisationId, organisationSource).First(&org).Error
if err != nil {
log.Printf("could not fetch organisation: %v err: %v", organisationId, err)
c.JSON(http.StatusNotFound, gin.H{"error": "Could not fetch organisation"})
return
}

var connections []models.VCSConnection
err = models.DB.GormDB.Where("organisation_id = ?", org.ID).Find(&connections).Error
if err != nil {
log.Printf("could not fetch VCS connections: %v", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "Could not fetch VCS connections"})
return
}

connectionsSlim := lo.Map(connections, func(c models.VCSConnection, i int) gin.H {
return gin.H{
"connection_id": c.ID,
"vcs": "bitbucket",
"connection_name": c.Name,
}
})
c.JSON(http.StatusOK, gin.H{
"result": connectionsSlim,
})
}

func CreateVCSConnectionApi(c *gin.Context) {
organisationId := c.GetString(middleware.ORGANISATION_ID_KEY)
organisationSource := c.GetString(middleware.ORGANISATION_SOURCE_KEY)

var org models.Organisation
err := models.DB.GormDB.Where("external_id = ? AND external_source = ?", organisationId, organisationSource).First(&org).Error
if err != nil {
log.Printf("could not fetch organisation: %v err: %v", organisationId, err)
c.JSON(http.StatusNotFound, gin.H{"error": "Could not fetch organisation"})
return
}

type CreateVCSConnectionRequest struct {
VCS string `json:"type" binding:"required"`
Name string `json:"connection_name"`
BitbucketAccessToken string `json:"bitbucket_access_token"`
BitbucketWebhookSecret string `json:"bitbucket_webhook_secret"`
}

var request CreateVCSConnectionRequest
if err := c.BindJSON(&request); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request body"})
return
}

if request.VCS != "bitbucket" {
log.Printf("VCS type not supported: %v", request.VCS)
c.JSON(http.StatusBadRequest, gin.H{"error": "VCS type not supported"})
return
}

secret := os.Getenv("DIGGER_ENCRYPTION_SECRET")
if secret == "" {
log.Printf("ERROR: no encryption secret specified")
c.JSON(http.StatusInternalServerError, gin.H{"error": "Could not encrypt access token"})
return
}

bitbucketAccessTokenEncrypted, err := utils.AESEncrypt([]byte(secret), request.BitbucketAccessToken)
if err != nil {
log.Printf("could not encrypt access token: %v", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "Could not encrypt access token"})
return
}

bitbucketWebhookSecretEncrypted, err := utils.AESEncrypt([]byte(secret), request.BitbucketWebhookSecret)
if err != nil {
log.Printf("could not encrypt webhook secret: %v", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "Could not encrypt webhook secret"})
return
}

connection, err := models.DB.CreateVCSConnection(
request.Name,
0,
"",
"",
"",
"",
"",
"",
"",
bitbucketAccessTokenEncrypted,
bitbucketWebhookSecretEncrypted,
org.ID,
)
if err != nil {
log.Printf("")
}

c.JSON(http.StatusCreated, gin.H{
"connection": connection.ID,
})
}

func GetVCSConnection(c *gin.Context) {
organisationId := c.GetString(middleware.ORGANISATION_ID_KEY)
organisationSource := c.GetString(middleware.ORGANISATION_SOURCE_KEY)
connectionId := c.Param("id")

var org models.Organisation
err := models.DB.GormDB.Where("external_id = ? AND external_source = ?", organisationId, organisationSource).First(&org).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
c.String(http.StatusNotFound, "Could not find organisation: "+organisationId)
} else {
log.Printf("could not fetch organisation: %v err: %v", organisationId, err)
c.String(http.StatusNotFound, "Could not fetch organisation: "+organisationId)
}
return
}

var connection models.VCSConnection
err = models.DB.GormDB.Where("id = ? AND organisation_id = ?", connectionId, org.ID).First(&connection).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
c.String(http.StatusNotFound, "Could not find connection: "+connectionId)
} else {
log.Printf("could not fetch connection: %v err: %v", connectionId, err)
c.String(http.StatusInternalServerError, "Could not fetch connection")
}
return
}

c.JSON(http.StatusOK, gin.H{
"connection_name": connection.Name,
"connection_id": connection.ID,
})
}

func DeleteVCSConnection(c *gin.Context) {
organisationId := c.GetString(middleware.ORGANISATION_ID_KEY)
organisationSource := c.GetString(middleware.ORGANISATION_SOURCE_KEY)
connectionId := c.Param("id")

var org models.Organisation
err := models.DB.GormDB.Where("external_id = ? AND external_source = ?", organisationId, organisationSource).First(&org).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
c.String(http.StatusNotFound, "Could not find organisation: "+organisationId)
} else {
log.Printf("could not fetch organisation: %v err: %v", organisationId, err)
c.String(http.StatusNotFound, "Could not fetch organisation: "+organisationId)
}
return
}

var connection models.VCSConnection
err = models.DB.GormDB.Where("id = ? AND organisation_id = ?", connectionId, org.ID).First(&connection).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
c.String(http.StatusNotFound, "Could not find connection: "+connectionId)
} else {
log.Printf("could not fetch connection: %v err: %v", connectionId, err)
c.String(http.StatusInternalServerError, "Could not fetch connection")
}
return
}

err = models.DB.GormDB.Delete(&connection).Error
if err != nil {
log.Printf("could not delete connection: %v err: %v", connectionId, err)
c.String(http.StatusInternalServerError, "Could not delete connection")
return
}

c.JSON(http.StatusOK, gin.H{
"status": "success",
})
}
6 changes: 3 additions & 3 deletions backend/controllers/github.go
Original file line number Diff line number Diff line change
Expand Up @@ -522,7 +522,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
aiSummaryCommentId = aiSummaryComment.Id
}

batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGithub, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs)
batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGithub, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs, nil)
if err != nil {
log.Printf("ConvertJobsToDiggerJobs error: %v", err)
commentReporterManager.UpdateComment(fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))
Expand Down Expand Up @@ -595,7 +595,7 @@ func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6
var diggerYmlStr string
var dependencyGraph graph.Graph[string, dg_configuration.Project]

err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, func(dir string) error {
err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error {
diggerYmlStr, err = dg_configuration.ReadDiggerYmlFileContents(dir)
if err != nil {
log.Printf("could not load digger config: %v", err)
Expand Down Expand Up @@ -930,7 +930,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
aiSummaryCommentId = aiSummaryComment.Id
}

batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, *commitSha, reporterCommentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs)
batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, *commitSha, reporterCommentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs, nil)
if err != nil {
log.Printf("ConvertJobsToDiggerJobs error: %v", err)
commentReporterManager.UpdateComment(fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))
Expand Down
10 changes: 5 additions & 5 deletions backend/controllers/github_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -589,7 +589,7 @@ func setupSuite(tb testing.TB) (func(tb testing.TB), *models.Database) {

// migrate tables
err = gdb.AutoMigrate(&models.Policy{}, &models.Organisation{}, &models.Repo{}, &models.Project{}, &models.Token{},
&models.User{}, &models.ProjectRun{}, &models.GithubAppInstallation{}, &models.GithubAppConnection{}, &models.GithubAppInstallationLink{},
&models.User{}, &models.ProjectRun{}, &models.GithubAppInstallation{}, &models.VCSConnection{}, &models.GithubAppInstallationLink{},
&models.GithubDiggerJobLink{}, &models.DiggerJob{}, &models.DiggerJobParentLink{}, &models.JobToken{})
if err != nil {
log.Fatal(err)
Expand Down Expand Up @@ -731,7 +731,7 @@ func TestJobsTreeWithOneJobsAndTwoProjects(t *testing.T) {
graph, err := configuration.CreateProjectDependencyGraph(projects)
assert.NoError(t, err)

_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 41584295, "", 2, "diggerhq", "parallel_jobs_demo", "diggerhq/parallel_jobs_demo", "", 123, "test", 0, "", false)
_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 41584295, "", 2, "diggerhq", "parallel_jobs_demo", "diggerhq/parallel_jobs_demo", "", 123, "test", 0, "", false, nil)
assert.NoError(t, err)
assert.Equal(t, 1, len(result))
parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["dev"].DiggerJobID)
Expand Down Expand Up @@ -760,7 +760,7 @@ func TestJobsTreeWithTwoDependantJobs(t *testing.T) {
projectMap["dev"] = project1
projectMap["prod"] = project2

_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false)
_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, nil)
assert.NoError(t, err)
assert.Equal(t, 2, len(result))

Expand Down Expand Up @@ -793,7 +793,7 @@ func TestJobsTreeWithTwoIndependentJobs(t *testing.T) {
projectMap["dev"] = project1
projectMap["prod"] = project2

_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false)
_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, nil)
assert.NoError(t, err)
assert.Equal(t, 2, len(result))
parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["dev"].DiggerJobID)
Expand Down Expand Up @@ -838,7 +838,7 @@ func TestJobsTreeWithThreeLevels(t *testing.T) {
projectMap["555"] = project5
projectMap["666"] = project6

_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false)
_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, nil)
assert.NoError(t, err)
assert.Equal(t, 6, len(result))
parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["111"].DiggerJobID)
Expand Down
4 changes: 2 additions & 2 deletions backend/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ replace github.com/ugorji/go => github.com/ugorji/go v1.2.12
require (
ariga.io/atlas-provider-gorm v0.5.0
github.com/bradleyfalzon/ghinstallation/v2 v2.11.0
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
github.com/dchest/uniuri v1.2.0
github.com/diggerhq/digger/libs v0.4.15
github.com/dominikbraun/graph v0.23.0
Expand All @@ -20,6 +21,7 @@ require (
github.com/golang-jwt/jwt v3.2.2+incompatible
github.com/google/go-github/v61 v61.0.0
github.com/google/uuid v1.6.0
github.com/ktrysmt/go-bitbucket v0.9.81
github.com/migueleliasweb/go-github-mock v0.0.23
github.com/robfig/cron v1.2.0
github.com/samber/lo v1.39.0
Expand All @@ -29,7 +31,6 @@ require (
github.com/stretchr/testify v1.9.0
github.com/xanzy/go-gitlab v0.106.0
golang.org/x/oauth2 v0.24.0
gorm.io/datatypes v1.2.4
gorm.io/driver/postgres v1.5.7
gorm.io/driver/sqlite v1.5.5
gorm.io/gorm v1.25.11
Expand Down Expand Up @@ -119,7 +120,6 @@ require (
github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
github.com/creack/pty v1.1.17 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dimchansky/utfbom v1.1.1 // indirect
github.com/dineshba/tf-summarize v0.3.10 // indirect
github.com/envoyproxy/go-control-plane v0.13.0 // indirect
Expand Down
Loading

0 comments on commit 3f0a92b

Please sign in to comment.