Use custom package_paths in EPR (elastic#104)

* Use multiple package paths

* Don't pull package-storage

* Fix: package paths

* Fix: mage check

config.yml

@@ -0,0 +1,2 @@
+package_paths:
+  - ./build/integrations

go.mod

@@ -4,7 +4,7 @@ go 1.12
 
 require (
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/elastic/package-registry v0.4.1-0.20200623125513-046078adc150
+	github.com/elastic/package-registry v0.4.1-0.20200624083400-5b436bf3479b
 	github.com/magefile/mage v1.9.0
 	github.com/pkg/errors v0.9.1
 	gopkg.in/yaml.v2 v2.3.0

go.sum

@@ -8,8 +8,8 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/elastic/go-ucfg v0.8.4-0.20200415140258-1232bd4774a6 h1:Ehbr7du4rSSEypR8zePr0XRbMhO4PJgcHC9f8fDbgAg=
 github.com/elastic/go-ucfg v0.8.4-0.20200415140258-1232bd4774a6/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo=
-github.com/elastic/package-registry v0.4.1-0.20200623125513-046078adc150 h1:bNeQkZLto6i6cJgUGCXUrnuQ8IpN6qB5VjIwNLxXDT8=
-github.com/elastic/package-registry v0.4.1-0.20200623125513-046078adc150/go.mod h1:oQx3Tg9ynuC6APd0o0OHud9kyPX6S6IzdJp/R4Hj1HY=
+github.com/elastic/package-registry v0.4.1-0.20200624083400-5b436bf3479b h1:W044lkpjc+GojaciFszMJuXOrUs6w4aZTUQfP/H16mY=
+github.com/elastic/package-registry v0.4.1-0.20200624083400-5b436bf3479b/go.mod h1:oQx3Tg9ynuC6APd0o0OHud9kyPX6S6IzdJp/R4Hj1HY=
 github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
 github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4=

magefile.go

@@ -22,21 +22,13 @@ import (
 )
 
 var (
-	// GoImportsImportPath controls the import path used to install goimports.
-	GoImportsImportPath = "golang.org/x/tools/cmd/goimports"
-
 	// GoImportsLocalPrefix is a string prefix matching imports that should be
 	// grouped after third-party packages.
 	GoImportsLocalPrefix = "github.com/elastic"
 
-	// GoLicenserImportPath controls the import path used to install go-licenser.
-	GoLicenserImportPath = "github.com/elastic/go-licenser"
-
-	buildDir           = "./build"
-	publicDir          = filepath.Join(buildDir, "public")
-	storageRepoDir     = filepath.Join(buildDir, "package-storage")
-	storagePackagesDir = filepath.Join(buildDir, "package-storage-packages")
-	packagePaths       = []string{storagePackagesDir, "./packages"}
+	buildDir             = "./build"
+	integrationsDir      = "./packages"
+	integrationsBuildDir = filepath.Join(buildDir, "integrations")
 
 	fieldsToEncode = []string{
 		"attributes.kibanaSavedObjectMeta.searchSourceJSON",
@@ -55,17 +47,12 @@ type fieldEntry struct {
 }
 
 func Build() error {
-	err := buildPublicDirectory()
-	if err != nil {
-		return err
-	}
-
-	err = fetchPackageStorage()
+	err := prepareBuildDirectory()
 	if err != nil {
 		return err
 	}
 
-	err = buildPackages()
+	err = buildIntegrations()
 	if err != nil {
 		return err
 	}
@@ -77,62 +64,28 @@ func Build() error {
 	return nil
 }
 
-func buildPublicDirectory() error {
-	err := os.MkdirAll(publicDir, 0755)
+func prepareBuildDirectory() error {
+	err := os.MkdirAll(integrationsBuildDir, 0755)
 	if err != nil {
 		return err
 	}
 
-	err = os.RemoveAll(filepath.Join(publicDir, "package"))
+	contents, err := ioutil.ReadDir(integrationsBuildDir)
 	if err != nil {
 		return err
 	}
-	return nil
-}
 
-func fetchPackageStorage() error {
-	_, err := os.Stat(storagePackagesDir)
-	if err == nil {
-		return nil // package storage has been already fetched
-	}
-
-	err = sh.Run("git", "clone", "https://github.com/elastic/package-storage.git", storageRepoDir)
-	if err != nil {
-		return err
-	}
-
-	packageStorageRevision := os.Getenv("PACKAGE_STORAGE_REVISION")
-	if packageStorageRevision == "" {
-		packageStorageRevision = "master"
-	}
-
-	err = sh.Run("git",
-		"--git-dir", filepath.Join(storageRepoDir, ".git"),
-		"--work-tree", storageRepoDir,
-		"checkout",
-		packageStorageRevision)
-	if err != nil {
-		return err
-	}
-
-	err = os.MkdirAll(storagePackagesDir, 0755)
-	if err != nil {
-		return err
-	}
-
-	err = sh.Run("rsync", "-a",
-		filepath.Join(storageRepoDir, "packages", "base")+"/",
-		filepath.Join(storagePackagesDir, "base"))
-	if err != nil {
-		return err
+	for _, c := range contents {
+		err = os.RemoveAll(filepath.Join(integrationsBuildDir, c.Name()))
+		if err != nil {
+			return err
+		}
 	}
-	return sh.Run("rsync", "-a",
-		filepath.Join(storageRepoDir, "packages", "endpoint")+"/",
-		filepath.Join(storagePackagesDir, "endpoint"))
+	return nil
 }
 
-func buildPackages() error {
-	packagePaths, err := findPackages()
+func buildIntegrations() error {
+	packagePaths, err := findIntegrations()
 	if err != nil {
 		return err
 	}
@@ -143,7 +96,7 @@ func buildPackages() error {
 		if err != nil {
 			return err
 		}
-		dstDir := filepath.Join(publicDir, "package", p.Name, p.Version)
+		dstDir := filepath.Join(integrationsBuildDir, p.Name, p.Version)
 
 		err = copyPackageFromSource(srcDir, dstDir)
 		if err != nil {
@@ -158,34 +111,33 @@ func buildPackages() error {
 	return nil
 }
 
-func findPackages() ([]string, error) {
+func findIntegrations() ([]string, error) {
 	var matches []string
-	for _, sourceDir := range packagePaths {
-		err := filepath.Walk(sourceDir, func(path string, info os.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
-
-			f, err := os.Stat(path)
-			if err != nil {
-				return err
-			}
 
-			if !f.IsDir() {
-				return nil // skip as the path is not a directory
-			}
+	err := filepath.Walk(integrationsDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
 
-			manifestPath := filepath.Join(path, "manifest.yml")
-			_, err = os.Stat(manifestPath)
-			if os.IsNotExist(err) {
-				return nil
-			}
-			matches = append(matches, path)
-			return filepath.SkipDir
-		})
+		f, err := os.Stat(path)
 		if err != nil {
-			return nil, err
+			return err
+		}
+
+		if !f.IsDir() {
+			return nil // skip as the path is not a directory
 		}
+
+		manifestPath := filepath.Join(path, "manifest.yml")
+		_, err = os.Stat(manifestPath)
+		if os.IsNotExist(err) {
+			return nil
+		}
+		matches = append(matches, path)
+		return filepath.SkipDir
+	})
+	if err != nil {
+		return nil, err
 	}
 	return matches, nil
 }
@@ -270,18 +222,7 @@ func encodedSavedObject(data []byte) ([]byte, bool, error) {
 }
 
 func dryRunPackageRegistry() error {
-	currentDir, err := os.Getwd()
-	if err != nil {
-		return errors.Wrap(err, "reading current directory failed")
-	}
-	defer os.Chdir(currentDir)
-
-	err = os.Chdir(buildDir)
-	if err != nil {
-		return errors.Wrapf(err, "can't change directory to %s", buildDir)
-	}
-
-	err = sh.Run("go", "run", "github.com/elastic/package-registry", "-dry-run=true")
+	err := sh.Run("go", "run", "github.com/elastic/package-registry", "-dry-run=true")
 	if err != nil {
 		return errors.Wrap(err, "package-registry dry-run failed")
 	}

packages/mysql/manifest.yml

@@ -3,7 +3,7 @@ name: mysql
 title: MySQL
 version: 0.1.3
 license: basic
-description: MySQL Integration
+description: MySQL Integration-7
 type: integration
 categories:
 - logs

testing/environments/package-registry.config.yml

@@ -0,0 +1,4 @@
+package_paths:
+  - /registry/packages/integrations
+  - /registry/packages/package-storage
+dev_mode: true

testing/environments/snapshot.yml

@@ -44,6 +44,7 @@ services:
       retries: 300
       interval: 1s
     volumes:
-      - ../../build/public:/registry/public
+      - ./package-registry.config.yml:/registry/config.yml
+      - ../../build/integrations:/registry/packages/integrations
     ports:
       - "127.0.0.1:8080:8080"