Multi plugins, plugin .conf filles (zalando#598)

* load multiple types of plugins from one module
so github.com/aryszka/configfilter could be converted to a plugin
* add example multiplugin code
* update
* config files / multitype plugins / plugin tests
* when loading a .so file, it is checked if there is also a .conf file
  with the same base, e.g. for `./plugins/noop.so` a config of
  `./plugins/noop.conf` is attempted to open. If it exists, each line
  in this file is passed as argument to the plugin loader function
  (leading and trailing spaces stripped, lines beginning with a `#`
  after space stripping are ignored).
* add support for multitype plugins (this time with complete auto
  detection and `.conf` file support)
* add some noop plugins in _test_plugins and add a make target to build
  and try loading as test
* run check-plugins target on each travis run
* drop unused stuff
* unify
* switch InitPlugin signature

InitPlugin is now

```
InitPlugin(opts []string) ([]filters.Spec, []routing.Predicate, []routing.DataClient, error)
```

also:
* any plugin which implements multiple Init* causes an error
* use logrus
* update .gitignore
* ignore _test_plugins_fail
* update docs
* check-plugins also in shortcheck

Author: vetinari

.gitignore

@@ -17,3 +17,5 @@ vendor
 linter.log
 site/
 plugins/
+_test_plugins/*.so
+_test_plugins_fail/*.so

Makefile

@@ -1,4 +1,4 @@
-SOURCES            = $(shell find . -name '*.go' -not -path "./vendor/*")
+SOURCES            = $(shell find . -name '*.go' -not -path "./vendor/*" -and -not -path "./_test_plugins" -and -not -path "./_test_plugins_fail" )
 PACKAGES           = $(shell glide novendor || echo -n "./...")
 CURRENT_VERSION    = $(shell git describe --tags --always --dirty)
 VERSION           ?= $(CURRENT_VERSION)
@@ -7,6 +7,11 @@ NEXT_MINOR         = $(shell go run packaging/version/version.go minor $(CURRENT
 NEXT_PATCH         = $(shell go run packaging/version/version.go patch $(CURRENT_VERSION))
 COMMIT_HASH        = $(shell git rev-parse --short HEAD)
 TEST_ETCD_VERSION ?= v2.3.8
+TEST_PLUGINS       = _test_plugins/filter_noop.so \
+		     _test_plugins/predicate_match_none.so \
+		     _test_plugins/dataclient_noop.so \
+		     _test_plugins/multitype_noop.so \
+		     _test_plugins_fail/fail.so
 
 default: build
 
@@ -34,22 +39,31 @@ install: $(SOURCES)
 	go install -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" ./cmd/skipper
 	go install -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" ./cmd/eskip
 
-check: build
+check: build check-plugins
 	# go test $(PACKAGES)
 	#
 	# due to vendoring and how go test ./... is not the same as go test ./a/... ./b/...
 	# probably can be reverted once etcd is fully mocked away for tests
 	#
 	for p in $(PACKAGES); do go test $$p || break; done
 
-shortcheck: build
+shortcheck: build check-plugins
 	# go test -test.short -run ^Test $(PACKAGES)
 	#
 	# due to vendoring and how go test ./... is not the same as go test ./a/... ./b/...
 	# probably can be reverted once etcd is fully mocked away for tests
 	#
 	for p in $(PACKAGES); do go test -test.short -run ^Test $$p || break -1; done
 
+check-plugins: $(TEST_PLUGINS)
+	go test -run LoadPlugins
+
+_test_plugins/%.so: _test_plugins/%.go
+	go build -buildmode=plugin -o $@ $<
+
+_test_plugins_fail/%.so: _test_plugins_fail/%.go
+	go build -buildmode=plugin -o $@ $<
+
 bench: build
 	# go test -bench . $(PACKAGES)
 	#
@@ -64,6 +78,8 @@ lint: build
 clean:
 	go clean -i ./...
 	rm -rf .coverprofile-all .cover
+	rm ./_test_plugins/*.so
+	rm ./_test_plugins_fail/*.so
 
 deps:
 	go get -t github.com/zalando/skipper/...
@@ -156,5 +172,5 @@ else ifeq ($(TRAVIS_BRANCH)_$(TRAVIS_PULL_REQUEST), master_false)
 else ifeq ($(TRAVIS_BRANCH), master)
 	make deps check-precommit
 else
-	make deps shortcheck
+	make deps shortcheck check-plugins
 endif

_test_plugins/dataclient_noop.go

@@ -0,0 +1,21 @@
+package main
+
+import (
+	"github.com/zalando/skipper/eskip"
+	"github.com/zalando/skipper/routing"
+)
+
+type DataClient string
+
+func InitDataClient([]string) (routing.DataClient, error) {
+	var dc DataClient = ""
+	return dc, nil
+}
+
+func (dc DataClient) LoadAll() ([]*eskip.Route, error) {
+	return eskip.Parse(string(dc))
+}
+
+func (dc DataClient) LoadUpdate() ([]*eskip.Route, []string, error) {
+	return nil, nil, nil
+}

_test_plugins/filter_noop.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"github.com/zalando/skipper/filters"
+)
+
+type noopSpec struct{}
+
+func InitFilter(opts []string) (filters.Spec, error) {
+	return noopSpec{}, nil
+}
+
+func (s noopSpec) Name() string {
+	return "noop"
+}
+func (s noopSpec) CreateFilter(config []interface{}) (filters.Filter, error) {
+	return noopFilter{}, nil
+}
+
+type noopFilter struct{}
+
+func (f noopFilter) Request(filters.FilterContext)  {}
+func (f noopFilter) Response(filters.FilterContext) {}

_test_plugins/multitype_noop.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"net/http"
+
+	"github.com/zalando/skipper/filters"
+	"github.com/zalando/skipper/routing"
+)
+
+type noopSpec struct {
+	Type string
+}
+
+func InitPlugin(opts []string) ([]filters.Spec, []routing.PredicateSpec, []routing.DataClient, error) {
+	return []filters.Spec{noopSpec{"noop"}}, []routing.PredicateSpec{noopSpec{"None"}}, nil, nil
+}
+
+func (s noopSpec) Name() string {
+	return s.Type
+}
+
+func (s noopSpec) CreateFilter(config []interface{}) (filters.Filter, error) {
+	return noop{}, nil
+}
+
+func (s noopSpec) Create(config []interface{}) (routing.Predicate, error) {
+	return noop{}, nil
+}
+
+type noop struct{}
+
+func (p noop) Request(filters.FilterContext)  {}
+func (p noop) Response(filters.FilterContext) {}
+
+func (p noop) Match(*http.Request) bool {
+	return false
+}

_test_plugins/predicate_match_none.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"net/http"
+
+	"github.com/zalando/skipper/routing"
+)
+
+type noneSpec struct{}
+
+func InitPredicate(opts []string) (routing.PredicateSpec, error) {
+	return noneSpec{}, nil
+}
+
+func (s noneSpec) Name() string {
+	return "None"
+}
+func (s noneSpec) Create(config []interface{}) (routing.Predicate, error) {
+	return nonePredicate{}, nil
+}
+
+type nonePredicate struct{}
+
+func (p nonePredicate) Match(*http.Request) bool {
+	return false
+}

_test_plugins_fail/fail.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"net/http"
+
+	"github.com/zalando/skipper/filters"
+	"github.com/zalando/skipper/routing"
+)
+
+// this fails to load, because it implements multiple Init* functions
+
+func InitFilter(opts []string) (filters.Spec, error) {
+	return noopSpec{}, nil
+}
+
+func InitPredicate(opts []string) (routing.PredicateSpec, error) {
+	return noneSpec{}, nil
+}
+
+type noopSpec struct{}
+
+func (s noopSpec) Name() string {
+	return "noop"
+}
+func (s noopSpec) CreateFilter(config []interface{}) (filters.Filter, error) {
+	return noopFilter{}, nil
+}
+
+type noopFilter struct{}
+
+func (f noopFilter) Request(filters.FilterContext)  {}
+func (f noopFilter) Response(filters.FilterContext) {}
+
+type noneSpec struct{}
+
+func (s noneSpec) Name() string {
+	return "None"
+}
+func (s noneSpec) Create(config []interface{}) (routing.Predicate, error) {
+	return nonePredicate{}, nil
+}
+
+type nonePredicate struct{}
+
+func (p nonePredicate) Match(*http.Request) bool {
+	return false
+}

cmd/skipper/main.go

@@ -211,6 +211,7 @@ var (
 	filterPlugins                   pluginFlags
 	predicatePlugins                pluginFlags
 	dataclientPlugins               pluginFlags
+	multiPlugins                    pluginFlags
 )
 
 func init() {
@@ -295,6 +296,7 @@ func init() {
 	flag.Var(&filterPlugins, "filter-plugin", filterPluginUsage)
 	flag.Var(&predicatePlugins, "predicate-plugin", predicatePluginUsage)
 	flag.Var(&dataclientPlugins, "dataclient-plugin", dataclientPluginUsage)
+	flag.Var(&multiPlugins, "multi-plugin", multiPluginUsage)
 
 	flag.Parse()
 
@@ -418,6 +420,7 @@ func main() {
 		FilterPlugins:                       filterPlugins.Get(),
 		PredicatePlugins:                    predicatePlugins.Get(),
 		DataClientPlugins:                   dataclientPlugins.Get(),
+		Plugins:                             multiPlugins.Get(),
 	}
 
 	if pluginDir != "" {

cmd/skipper/filterFlags.go cmd/skipper/pluginFlags.go

@@ -8,6 +8,7 @@ const (
 	filterPluginUsage     = "set a custom filter plugins to load, a comma separated list of name and arguments"
 	predicatePluginUsage  = "set a custom predicate plugins to load, a comma separated list of name and arguments"
 	dataclientPluginUsage = "set a custom dataclient plugins to load, a comma separated list of name and arguments"
+	multiPluginUsage      = "set a custom multitype plugins to load, a comma separated list of name and arguments"
 )
 
 type pluginFlags struct {