Merge pull request #218 from stelligent/develop

v1.1.1

.gitignore

@@ -1,6 +1,8 @@
 *.iml
 .idea
+.vscode
 .release/
 vendor/
 *~
-homebrew-tap/
+homebrew-tap/
+templates/assets.go

Makefile

@@ -16,6 +16,8 @@ UPLOAD_FILES = $(foreach os, $(TARGET_OS), $(PACKAGE)-$(os)-$(ARCH))
 GOLDFLAGS = "-X main.version=$(VERSION)"
 TAG_VERSION = v$(VERSION)
 
+export PATH := $(GOPATH)/bin:$(PATH)
+
 default: all
 
 deps:
@@ -26,7 +28,12 @@ deps:
 	go get "github.com/aktau/github-release"
 	glide install
 	patch -p1 < go-git.v4.patch
+
+ifeq ($(CIRCLECI),true)
 	gem install cfn-nag
+else
+	gem list | grep cfn-nag || sudo gem install cfn-nag
+endif
 
 gen:
 	go generate $(SRC_FILES)
@@ -55,9 +62,9 @@ test: lint gen nag
 	@echo "=== testing ==="
 ifneq ($(CIRCLE_WORKING_DIRECTORY),)
 	mkdir -p $(CIRCLE_WORKING_DIRECTORY)/test-results/unit
-	go test -v -cover $(SRC_FILES) -short | go-junit-report > $(CIRCLE_WORKING_DIRECTORY)/test-results/unit/report.xml
+	go test -v -cover $(filter-out ./e2e/..., $(SRC_FILES)) -short | go-junit-report > $(CIRCLE_WORKING_DIRECTORY)/test-results/unit/report.xml
 else
-	go test -cover $(SRC_FILES) -short
+	go test -cover $(filter-out ./e2e/..., $(SRC_FILES)) -short
 endif
 
 e2e: gen stage keypair
@@ -160,5 +167,8 @@ fmt:
 	@echo "=== formatting ==="
 	go fmt $(SRC_FILES)
 
+changelog:
+	github_changelog_generator -u stelligent -p mu -t $(GITHUB_TOKEN)
+
 
 .PHONY: default all lint test e2e build deps gen clean release-clean release-create dev-release release install $(UPLOAD_FILES) $(BUILD_FILES) $(TARGET_OS) keypair stage

VERSION

@@ -1 +1 @@
-1.0.6
+1.1.1

cli/app.go

@@ -1,6 +1,7 @@
 package cli
 
 import (
+	"fmt"
 	"github.com/stelligent/mu/common"
 	"github.com/stelligent/mu/provider/aws"
 	"github.com/urfave/cli"
@@ -44,7 +45,12 @@ func NewApp() *cli.App {
 		}
 
 		// TODO: support initializing context from other cloud providers?
-		err = aws.InitializeContext(context, c.String("profile"), c.String("assume-role"), c.String("region"), c.Bool("dryrun"), c.Bool("skip-version-check"))
+		log.Debugf("dryrun:%v path:%v", c.Bool("dryrun"), c.String("dryrun-output"))
+		dryrunPath := ""
+		if c.Bool("dryrun") {
+			dryrunPath = c.String("dryrun-output")
+		}
+		err = aws.InitializeContext(context, c.String("profile"), c.String("assume-role"), c.String("region"), dryrunPath, c.Bool("skip-version-check"))
 		if err != nil {
 			return err
 		}
@@ -86,8 +92,8 @@ func NewApp() *cli.App {
 			context.Config.Namespace = "mu"
 		}
 
-		return nil
-
+		// initialize extensions
+		return context.InitializeExtensions()
 	}
 
 	app.Flags = []cli.Flag{
@@ -124,6 +130,11 @@ func NewApp() *cli.App {
 			Name:  "dryrun, d",
 			Usage: "generate the cloudformation templates without upserting stacks",
 		},
+		cli.StringFlag{
+			Name:  "dryrun-output, O",
+			Usage: "output directory for dryrun",
+			Value: fmt.Sprintf("%s/mu-cloudformation", os.TempDir()),
+		},
 		cli.BoolFlag{
 			Name:  "disable-iam, I",
 			Usage: "disable the automatic creation of IAM resources",

cli/app_test.go

@@ -14,7 +14,7 @@ func TestNewApp(t *testing.T) {
 	assert.Equal("1.0.0-local", app.Version, "Version should match")
 	assert.Equal("Microservice Platform on AWS", app.Usage, "usage should match")
 	assert.Equal(true, app.EnableBashCompletion, "bash completion should match")
-	assert.Equal(10, len(app.Flags), "Flags len should match")
+	assert.Equal(11, len(app.Flags), "Flags len should match")
 	assert.Equal("config, c", app.Flags[0].GetName(), "Flags name should match")
 	assert.Equal("region, r", app.Flags[1].GetName(), "Flags name should match")
 	assert.Equal("assume-role, a", app.Flags[2].GetName(), "Flags name should match")
@@ -23,8 +23,9 @@ func TestNewApp(t *testing.T) {
 	assert.Equal("silent, s", app.Flags[5].GetName(), "Flags name should match")
 	assert.Equal("verbose, V", app.Flags[6].GetName(), "Flags name should match")
 	assert.Equal("dryrun, d", app.Flags[7].GetName(), "Flags name should match")
-	assert.Equal("disable-iam, I", app.Flags[8].GetName(), "Flags name should match")
-	assert.Equal("skip-version-check, F", app.Flags[9].GetName(), "Flags name should match")
+	assert.Equal("dryrun-output, O", app.Flags[8].GetName(), "Flags name should match")
+	assert.Equal("disable-iam, I", app.Flags[9].GetName(), "Flags name should match")
+	assert.Equal("skip-version-check, F", app.Flags[10].GetName(), "Flags name should match")
 	assert.Equal(5, len(app.Commands), "Commands len should match")
 	assert.Equal("init", app.Commands[0].Name, "Command[0].name should match")
 	assert.Equal("environment", app.Commands[1].Name, "Command[1].name should match")

cli/pipelines.go

@@ -2,14 +2,15 @@ package cli
 
 import (
 	"fmt"
-	"github.com/stelligent/mu/common"
-	"github.com/stelligent/mu/workflows"
-	"github.com/urfave/cli"
-	"golang.org/x/crypto/ssh/terminal"
 	"os"
 	"strings"
 	"syscall"
 	"time"
+
+	"github.com/stelligent/mu/common"
+	"github.com/stelligent/mu/workflows"
+	"github.com/urfave/cli"
+	"golang.org/x/crypto/ssh/terminal"
 )
 
 func newPipelinesCommand(ctx *common.Context) *cli.Command {

cli/services.go

@@ -124,9 +124,10 @@ func newServicesRestartCommand(ctx *common.Context) *cli.Command {
 				Name:  ServiceFlag,
 				Usage: SvcRestartServiceFlagUsage,
 			},
-			cli.StringFlag{
+			cli.IntFlag{
 				Name:  BatchFlag,
 				Usage: SvcRestartBatchFlagUsage,
+				Value: 1,
 			},
 		},
 		Action: func(c *cli.Context) error {

common/artifact.go

@@ -1,13 +1,21 @@
 package common
 
-import "io"
+import (
+	"io"
+)
 
-// ArtifactCreator for getting cluster instances
+// ArtifactCreator for creating artifacts
 type ArtifactCreator interface {
 	CreateArtifact(body io.ReadSeeker, destURI string) error
 }
 
+// ArtifactGetter for getting artifacts.  conditional get (based on etag).  returns body, etag and optional error
+type ArtifactGetter interface {
+	GetArtifact(uri string, etag string) (io.ReadCloser, string, error)
+}
+
 // ArtifactManager composite of all artifact capabilities
 type ArtifactManager interface {
 	ArtifactCreator
+	ArtifactGetter
 }

common/context.go

@@ -3,8 +3,10 @@ package common
 import (
 	"bufio"
 	"bytes"
+	"fmt"
 	"gopkg.in/yaml.v2"
 	"io"
+	"net/url"
 	"os"
 	"path"
 	"path/filepath"
@@ -161,8 +163,60 @@ func (ctx *Context) InitializeConfig(configReader io.Reader) error {
 		return err
 	}
 
-	// register the stack overrides
-	registerStackOverrides(ctx.Config.Templates)
+	return nil
+}
+
+// InitializeExtensions loads extension objects
+func (ctx *Context) InitializeExtensions() error {
+	extMgr := ctx.ExtensionsManager
+	// load extensions from mu.yml
+	for _, extension := range ctx.Config.Extensions {
+		if extension.URL != "" {
+			u, err := parseAbsURL(extension.URL, ctx.Config.Basedir)
+			if err != nil {
+				log.Warningf("Unable to load extension '%s': %s", extension.URL, err)
+			} else {
+				ext, err := newTemplateArchiveExtension(u, ctx.ArtifactManager)
+				if err != nil {
+					log.Warningf("Unable to load extension '%s': %s", extension.URL, err)
+				} else {
+					err = extMgr.AddExtension(ext)
+					if err != nil {
+						log.Warningf("Unable to load extension '%s': %s", extension.URL, err)
+					}
+				}
+			}
+		} else if extension.Image != "" {
+			log.Warningf("Docker based extensions is not yet supported!")
+		}
+	}
+
+	// register the stack overrides from within the mu.yml
+	for stackName, template := range ctx.Config.Templates {
+		ext := newTemplateOverrideExtension(stackName, template)
+		err := extMgr.AddExtension(ext)
+		if err != nil {
+			log.Warningf("Unable to load extension '%s': %s", ext.ID(), err)
+		}
+	}
+
+	// register the stack parameters from within the mu.yml
+	for stackName, parameters := range ctx.Config.Parameters {
+		ext := newParameterOverrideExtension(stackName, parameters)
+		err := extMgr.AddExtension(ext)
+		if err != nil {
+			log.Warningf("Unable to load extension '%s': %s", ext.ID(), err)
+		}
+	}
+
+	// register the stack tags from within the mu.yml
+	for stackName, tags := range ctx.Config.Tags {
+		ext := newTagOverrideExtension(stackName, tags)
+		err := extMgr.AddExtension(ext)
+		if err != nil {
+			log.Warningf("Unable to load extension '%s': %s", ext.ID(), err)
+		}
+	}
 
 	return nil
 }
@@ -177,6 +231,12 @@ func (ctx *Context) InitializeContext() error {
 		return err
 	}
 
+	// initialize ExtensionsManager
+	ctx.ExtensionsManager, err = newExtensionsManager()
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -185,3 +245,20 @@ func loadYamlConfig(config *Config, yamlReader io.Reader) error {
 	yamlBuffer.ReadFrom(yamlReader)
 	return yaml.Unmarshal(yamlBuffer.Bytes(), config)
 }
+
+func parseAbsURL(urlString string, basedir string) (*url.URL, error) {
+	u, err := url.Parse(urlString)
+	if err != nil {
+		return nil, err
+	}
+
+	if !u.IsAbs() {
+		basedirURL, err := url.Parse(fmt.Sprintf("file://%s/", basedir))
+		if err != nil {
+			return nil, err
+		}
+		u = basedirURL.ResolveReference(u)
+		log.Debugf("Resolved relative path to '%s' from basedir '%s'", u, basedirURL)
+	}
+	return u, nil
+}