Github Enterprise support (denisenkom#234)

* exported Github struct fields and ReadDirectory method

* github ee implementation, tests and docs

* build fixes

* Github Enterprise API endpoint based on docs

* addressing PR comments

* code review

* make linter happy

* parseBool() takes fallback

* pr comments

* tweaks to Config{}

Author: kasparasg

Dockerfile

@@ -9,7 +9,7 @@ COPY . ./
 
 ENV GO111MODULE=on
 ENV DATABASES="postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb sqlserver"
-ENV SOURCES="file go_bindata github aws_s3 google_cloud_storage godoc_vfs gitlab"
+ENV SOURCES="file go_bindata github github_ee aws_s3 google_cloud_storage godoc_vfs gitlab"
 
 RUN go build -a -o build/migrate.linux-386 -ldflags="-X main.Version=${VERSION}" -tags "$DATABASES $SOURCES" ./cmd/migrate
 

Makefile

@@ -1,4 +1,4 @@
-SOURCE ?= file go_bindata github aws_s3 google_cloud_storage godoc_vfs gitlab
+SOURCE ?= file go_bindata github github_ee aws_s3 google_cloud_storage godoc_vfs gitlab
 DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb sqlserver
 VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-)
 TEST_FLAGS ?=
@@ -33,7 +33,7 @@ test:
 
 
 test-with-flags:
-	@echo SOURCE: $(SOURCE) 
+	@echo SOURCE: $(SOURCE)
 	@echo DATABASE: $(DATABASE)
 
 	@go test $(TEST_FLAGS) .
@@ -84,7 +84,7 @@ rewrite-import-paths:
 docs:
 	-make kill-docs
 	nohup godoc -play -http=127.0.0.1:6064 </dev/null >/dev/null 2>&1 & echo $$! > .godoc.pid
-	cat .godoc.pid  
+	cat .godoc.pid
 
 
 kill-docs:

README.md

@@ -66,6 +66,7 @@ Source drivers read migrations from local or remote sources. [Add a new source?]
 * [Filesystem](source/file) - read from filesystem
 * [Go-Bindata](source/go_bindata) - read from embedded binary data ([jteeuwen/go-bindata](https://github.com/jteeuwen/go-bindata))
 * [Github](source/github) - read from remote Github repositories
+* [Github Enterprise](source/github_ee) - read from remote Github Enterprise repositories
 * [Gitlab](source/gitlab) - read from remote Gitlab repositories
 * [AWS S3](source/aws_s3) - read from Amazon Web Services S3
 * [Google Cloud Storage](source/google_cloud_storage) - read from Google Cloud Platform Storage

internal/cli/build_github_ee.go

@@ -0,0 +1,7 @@
+// +build github
+
+package cli
+
+import (
+	_ "github.com/golang-migrate/migrate/v4/source/github_ee"
+)

source/github/README.md

@@ -1,5 +1,7 @@
 # github
 
+This driver is catered for those that want to source migrations from [github.com](https://github.com). The URL scheme doesn't require a hostname, as it just simply defaults to `github.com`.
+
 `github://user:personal-access-token@owner/repo/path#ref`
 
 | URL Query  | WithInstance Config | Description |

source/github/github.go

@@ -29,17 +29,17 @@ var (
 )
 
 type Github struct {
-	client *github.Client
-	url    string
-
-	pathOwner  string
-	pathRepo   string
-	path       string
+	config     *Config
+	client     *github.Client
 	options    *github.RepositoryContentGetOptions
 	migrations *source.Migrations
 }
 
 type Config struct {
+	Owner string
+	Repo  string
+	Path  string
+	Ref   string
 }
 
 func (g *Github) Open(url string) (source.Driver, error) {
@@ -64,20 +64,21 @@ func (g *Github) Open(url string) (source.Driver, error) {
 
 	gn := &Github{
 		client:     github.NewClient(tr.Client()),
-		url:        url,
 		migrations: source.NewMigrations(),
 		options:    &github.RepositoryContentGetOptions{Ref: u.Fragment},
 	}
 
+	gn.ensureFields()
+
 	// set owner, repo and path in repo
-	gn.pathOwner = u.Host
+	gn.config.Owner = u.Host
 	pe := strings.Split(strings.Trim(u.Path, "/"), "/")
 	if len(pe) < 1 {
 		return nil, ErrInvalidRepo
 	}
-	gn.pathRepo = pe[0]
+	gn.config.Repo = pe[0]
 	if len(pe) > 1 {
-		gn.path = strings.Join(pe[1:], "/")
+		gn.config.Path = strings.Join(pe[1:], "/")
 	}
 
 	if err := gn.readDirectory(); err != nil {
@@ -90,16 +91,29 @@ func (g *Github) Open(url string) (source.Driver, error) {
 func WithInstance(client *github.Client, config *Config) (source.Driver, error) {
 	gn := &Github{
 		client:     client,
+		config:     config,
 		migrations: source.NewMigrations(),
+		options:    &github.RepositoryContentGetOptions{Ref: config.Ref},
 	}
+
 	if err := gn.readDirectory(); err != nil {
 		return nil, err
 	}
+
 	return gn, nil
 }
 
 func (g *Github) readDirectory() error {
-	fileContent, dirContents, _, err := g.client.Repositories.GetContents(context.Background(), g.pathOwner, g.pathRepo, g.path, g.options)
+	g.ensureFields()
+
+	fileContent, dirContents, _, err := g.client.Repositories.GetContents(
+		context.Background(),
+		g.config.Owner,
+		g.config.Repo,
+		g.config.Path,
+		g.options,
+	)
+
 	if err != nil {
 		return err
 	}
@@ -120,37 +134,58 @@ func (g *Github) readDirectory() error {
 	return nil
 }
 
+func (g *Github) ensureFields() {
+	if g.config == nil {
+		g.config = &Config{}
+	}
+}
+
 func (g *Github) Close() error {
 	return nil
 }
 
 func (g *Github) First() (version uint, er error) {
+	g.ensureFields()
+
 	if v, ok := g.migrations.First(); !ok {
-		return 0, &os.PathError{Op: "first", Path: g.path, Err: os.ErrNotExist}
+		return 0, &os.PathError{Op: "first", Path: g.config.Path, Err: os.ErrNotExist}
 	} else {
 		return v, nil
 	}
 }
 
 func (g *Github) Prev(version uint) (prevVersion uint, err error) {
+	g.ensureFields()
+
 	if v, ok := g.migrations.Prev(version); !ok {
-		return 0, &os.PathError{Op: fmt.Sprintf("prev for version %v", version), Path: g.path, Err: os.ErrNotExist}
+		return 0, &os.PathError{Op: fmt.Sprintf("prev for version %v", version), Path: g.config.Path, Err: os.ErrNotExist}
 	} else {
 		return v, nil
 	}
 }
 
 func (g *Github) Next(version uint) (nextVersion uint, err error) {
+	g.ensureFields()
+
 	if v, ok := g.migrations.Next(version); !ok {
-		return 0, &os.PathError{Op: fmt.Sprintf("next for version %v", version), Path: g.path, Err: os.ErrNotExist}
+		return 0, &os.PathError{Op: fmt.Sprintf("next for version %v", version), Path: g.config.Path, Err: os.ErrNotExist}
 	} else {
 		return v, nil
 	}
 }
 
 func (g *Github) ReadUp(version uint) (r io.ReadCloser, identifier string, err error) {
+	g.ensureFields()
+
 	if m, ok := g.migrations.Up(version); ok {
-		file, _, _, err := g.client.Repositories.GetContents(context.Background(), g.pathOwner, g.pathRepo, path.Join(g.path, m.Raw), g.options)
+		file, _, _, err := g.client.Repositories.GetContents(
+			context.Background(),
+			g.config.Owner,
+			g.config.Repo,
+			path.Join(g.config.Path, m.Raw),
+			g.options,
+		)
+
 		if err != nil {
 			return nil, "", err
 		}
@@ -162,12 +197,21 @@ func (g *Github) ReadUp(version uint) (r io.ReadCloser, identifier string, err e
 			return ioutil.NopCloser(strings.NewReader(r)), m.Identifier, nil
 		}
 	}
-	return nil, "", &os.PathError{Op: fmt.Sprintf("read version %v", version), Path: g.path, Err: os.ErrNotExist}
+	return nil, "", &os.PathError{Op: fmt.Sprintf("read version %v", version), Path: g.config.Path, Err: os.ErrNotExist}
 }
 
 func (g *Github) ReadDown(version uint) (r io.ReadCloser, identifier string, err error) {
+	g.ensureFields()
+
 	if m, ok := g.migrations.Down(version); ok {
-		file, _, _, err := g.client.Repositories.GetContents(context.Background(), g.pathOwner, g.pathRepo, path.Join(g.path, m.Raw), g.options)
+		file, _, _, err := g.client.Repositories.GetContents(
+			context.Background(),
+			g.config.Owner,
+			g.config.Repo,
+			path.Join(g.config.Path, m.Raw),
+			g.options,
+		)
+
 		if err != nil {
 			return nil, "", err
 		}
@@ -179,5 +223,5 @@ func (g *Github) ReadDown(version uint) (r io.ReadCloser, identifier string, err
 			return ioutil.NopCloser(strings.NewReader(r)), m.Identifier, nil
 		}
 	}
-	return nil, "", &os.PathError{Op: fmt.Sprintf("read version %v", version), Path: g.path, Err: os.ErrNotExist}
+	return nil, "", &os.PathError{Op: fmt.Sprintf("read version %v", version), Path: g.config.Path, Err: os.ErrNotExist}
 }

source/github_ee/.gitignore

@@ -0,0 +1 @@
+.github_test_secrets

source/github_ee/README.md

@@ -0,0 +1,21 @@
+# github ee
+
+## Github Enterprise Edition
+
+This driver is catered for those who run Github Enterprise under private infrastructure.
+
+The below URL scheme illustrates how to source migration files from Github Enterprise.
+
+Github client for Go requires API and Uploads endpoint hosts in order to create an instance of Github Enterprise Client. We're making an assumption that the API and Uploads are available under `https://api.*` and `https://uploads.*` respectively. [Github Enterprise Installation Guide](https://help.github.com/en/enterprise/2.15/admin/installation/enabling-subdomain-isolation) recommends that you enable Subdomain isolation feature.
+
+`github-ee://user:personal-access-token@host/owner/repo/path?verify-tls=true#ref`
+
+| URL Query  | WithInstance Config | Description |
+|------------|---------------------|-------------|
+| user | | The username of the user connecting |
+| personal-access-token | | Personal access token from your Github Enterprise instance |
+| owner | | the repo owner |
+| repo | | the name of the repository |
+| path | | path in repo to migrations |
+| ref | | (optional) can be a SHA, branch, or tag |
+| verify-tls | | (optional) defaults to `true`. This option sets `tls.Config.InsecureSkipVerify` accordingly |

source/github_ee/github_ee.go

@@ -0,0 +1,96 @@
+package github_ee
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net/http"
+	nurl "net/url"
+	"strconv"
+	"strings"
+
+	"github.com/golang-migrate/migrate/v4/source"
+	gh "github.com/golang-migrate/migrate/v4/source/github"
+	"github.com/google/go-github/github"
+)
+
+func init() {
+	source.Register("github-ee", &GithubEE{})
+}
+
+type GithubEE struct {
+	source.Driver
+}
+
+func (g *GithubEE) Open(url string) (source.Driver, error) {
+	verifyTLS := true
+
+	u, err := nurl.Parse(url)
+	if err != nil {
+		return nil, err
+	}
+
+	if o := u.Query().Get("verify-tls"); o != "" {
+		verifyTLS = parseBool(o, verifyTLS)
+	}
+
+	if u.User == nil {
+		return nil, gh.ErrNoUserInfo
+	}
+
+	password, ok := u.User.Password()
+	if !ok {
+		return nil, gh.ErrNoUserInfo
+	}
+
+	ghc, err := g.createGithubClient(u.Host, u.User.Username(), password, verifyTLS)
+	if err != nil {
+		return nil, err
+	}
+
+	pe := strings.Split(strings.Trim(u.Path, "/"), "/")
+
+	if len(pe) < 1 {
+		return nil, gh.ErrInvalidRepo
+	}
+
+	cfg := &gh.Config{
+		Owner: pe[0],
+		Repo:  pe[1],
+		Ref:   u.Fragment,
+	}
+
+	if len(pe) > 2 {
+		cfg.Path = strings.Join(pe[2:], "/")
+	}
+
+	i, err := gh.WithInstance(ghc, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return &GithubEE{Driver: i}, nil
+}
+
+func (g *GithubEE) createGithubClient(host, username, password string, verifyTLS bool) (*github.Client, error) {
+	tr := &github.BasicAuthTransport{
+		Username: username,
+		Password: password,
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: !verifyTLS},
+		},
+	}
+
+	apiHost := fmt.Sprintf("https://%s/api/v3", host)
+	uploadHost := fmt.Sprintf("https://uploads.%s", host)
+
+	return github.NewEnterpriseClient(apiHost, uploadHost, tr.Client())
+}
+
+func parseBool(val string, fallback bool) bool {
+	b, err := strconv.ParseBool(val)
+	if err != nil {
+		return fallback
+	}
+
+	return b
+}

source/github_ee/github_ee_test.go

@@ -0,0 +1,44 @@
+package github_ee
+
+import (
+	"net/http"
+	"net/http/httptest"
+	nurl "net/url"
+	"testing"
+)
+
+func Test(t *testing.T) {
+	ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/api/v3/repos/mattes/migrate_test_tmp/contents/test" {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+
+		if ref := r.URL.Query().Get("ref"); ref != "452b8003e7" {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+
+		_, err := w.Write([]byte("[]"))
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+	}))
+	defer ts.Close()
+
+	u, err := nurl.Parse(ts.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	g := &GithubEE{}
+	_, err = g.Open("github-ee://foo:bar@" + u.Host + "/mattes/migrate_test_tmp/test?verify-tls=false#452b8003e7")
+
+	if err != nil {
+		t.Fatal(err)
+	}
+}