From 20c06805de9bf74190475e7515b01fc8fa819022 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Tue, 21 Jan 2020 14:02:50 +0100 Subject: [PATCH 01/11] Operator to evict tainted pods --- cmd/evicter/README.md | 5 ++ cmd/evicter/controller.go | 177 ++++++++++++++++++++++++++++++++++++++ cmd/evicter/main.go | 102 ++++++++++++++++++++++ go.mod | 2 + go.sum | 12 +-- 5 files changed, 292 insertions(+), 6 deletions(-) create mode 100644 cmd/evicter/README.md create mode 100644 cmd/evicter/controller.go create mode 100644 cmd/evicter/main.go diff --git a/cmd/evicter/README.md b/cmd/evicter/README.md new file mode 100644 index 0000000..b5ad1d4 --- /dev/null +++ b/cmd/evicter/README.md @@ -0,0 +1,5 @@ +# Evict tainted pods after period +Operator that runs within k8s to find and evict tainted pods + +* `k-rails/tainted-timestamp` +* `k-rails/tainted-prevent-eviction` diff --git a/cmd/evicter/controller.go b/cmd/evicter/controller.go new file mode 100644 index 0000000..2f3e073 --- /dev/null +++ b/cmd/evicter/controller.go @@ -0,0 +1,177 @@ +package main + +import ( + "fmt" + "strconv" + "time" + + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog" +) + +type podProvisioner interface { + Evict(namespace, podName string) error +} + +type Controller struct { + podStore cache.Indexer + queue workqueue.RateLimitingInterface + informer cache.Controller + podProvisioner podProvisioner + incubationPeriodSeconds time.Duration + started time.Time +} + +func NewController(queue workqueue.RateLimitingInterface, indexer cache.Indexer, informer cache.Controller, podProvisioner podProvisioner, incubationPeriodSeconds int64) *Controller { + return &Controller{ + informer: informer, + podStore: indexer, + queue: queue, + podProvisioner: podProvisioner, + incubationPeriodSeconds: time.Duration(incubationPeriodSeconds) * time.Second, + } +} + +func (c *Controller) processNextItem() bool { + key, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(key) + + err := c.evictPod(key.(string)) + c.handleErr(err, key) + return true +} + +// evictPod is the business logic of the controller. it checks the the eviction rules and conditions before calling the pod provisioner. +func (c *Controller) evictPod(key string) error { + obj, exists, err := c.podStore.GetByKey(key) + switch { + case err != nil: + return err + case !exists: + return nil + } + pod := obj.(*v1.Pod) + if !canEvict(pod, c.incubationPeriodSeconds) { + return nil + } + + ns, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + return err + } + return c.podProvisioner.Evict(ns, name) +} + +const ( + annotationPreventEviction = "k-rails/tainted-prevent-eviction" + annotationTimestamp = "k-rails/tainted-timestamp" +) + +func canEvict(pod *v1.Pod, incubationPeriod time.Duration) bool { + val, ok := pod.Annotations[annotationPreventEviction] + if ok { + if val == "yes" || val == "true" { + return false + } + } + + val, ok = pod.Annotations[annotationTimestamp] + if ok { + i, err := strconv.ParseInt(val, 10, 64) + if err != nil { + // todo: log + return true + } + timestamp := time.Unix(i, 0) + if time.Since(timestamp) < incubationPeriod { + return false + } + } + return true +} + +const maxWorkerRetries = 5 + +// handleErr checks if an error happened and makes sure we will retry later. +func (c *Controller) handleErr(err error, key interface{}) { + if err == nil { + // Forget about the #AddRateLimited history of the key on every successful synchronization. + // This ensures that future processing of updates for this key is not delayed because of + // an outdated error history. + c.queue.Forget(key) + return + } + + // This controller retries 5 times if something goes wrong. After that, it stops trying. + if c.queue.NumRequeues(key) < maxWorkerRetries { + klog.Infof("Error syncing pod %v: %v", key, err) + + // Re-enqueue the key rate limited. Based on the rate limiter on the + // queue and the re-enqueue history, the key will be processed later again. + c.queue.AddRateLimited(key) + return + } + + c.queue.Forget(key) + // Report to an external entity that, even after several retries, we could not successfully process this key + runtime.HandleError(err) + klog.Infof("Dropping pod %q out of the queue: %v", key, err) +} + +const reconciliationTick = 30 * time.Second +const startupGracePeriod = 90 * time.Second + +func (c *Controller) Run(threadiness int, stopCh chan struct{}) { + defer runtime.HandleCrash() + + // Let the workers stop when we are done + defer c.queue.ShutDown() + klog.Info("Starting Pod controller") + + go c.informer.Run(stopCh) + + // Wait for all involved caches to be synced, before processing items from the queue is started + if !cache.WaitForCacheSync(stopCh, c.informer.HasSynced) { + runtime.HandleError(fmt.Errorf("timed out waiting for caches to sync")) + return + } + + for i := 0; i < threadiness; i++ { + go wait.Until(c.runWorker, time.Second, stopCh) + } + + wait.Until(func() { + if time.Since(c.started) < startupGracePeriod { + return + } + if err := c.doReconciliation(); err != nil { + klog.Errorf("Reconciliation failed: %s", err) + } + }, reconciliationTick, stopCh) + + klog.Info("Stopping Pod controller") +} + +func (c *Controller) runWorker() { + for c.processNextItem() { + } +} + +func (c *Controller) doReconciliation() error { + klog.Info("Reconciliation started") + for _, key := range c.podStore.ListKeys() { + if err := c.evictPod(key); err != nil { + return errors.Wrapf(err, "pod %q", key) + } + } + klog.Info("Reconciliation completed") + return nil +} diff --git a/cmd/evicter/main.go b/cmd/evicter/main.go new file mode 100644 index 0000000..d859ae3 --- /dev/null +++ b/cmd/evicter/main.go @@ -0,0 +1,102 @@ +package main + +import ( + "flag" + + v1 "k8s.io/api/core/v1" + policy "k8s.io/api/policy/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/typed/policy/v1beta1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog" +) + +func main() { + var ( + kubeconfig = flag.String("kubeconfig", "", "absolute path to the kubeconfig file: `/.kube/config`") + master = flag.String("master", "", "master url") + labelSelector = flag.String("label-selector", "tainted=true", "label selector to discover tainted pods") + terminationGracePeriodSeconds = flag.Int64("termination-grace-period", 30, "pod termination grace period in seconds") + taintedIncubationPeriodSeconds = flag.Int64("incubation-period", 24*60*60, "time in seconds a tainted pod can run before eviction") + ) + flag.Parse() + flag.Set("logtostderr", "true") // glog: no disk log + + config, err := clientcmd.BuildConfigFromFlags(*master, *kubeconfig) + if err != nil { + klog.Fatal(err) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + klog.Fatal(err) + } + podListWatcher := cache.NewFilteredListWatchFromClient(clientset.CoreV1().RESTClient(), "pods", metav1.NamespaceDefault, + func(options *metav1.ListOptions) { + options.LabelSelector = *labelSelector + }) + + queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) + // Bind the workqueue to a cache with the help of an informer. This way we make sure that + // whenever the cache is updated, the pod key is added to the workqueue. + // Note that when we finally process the item from the workqueue, we might see a newer version + // of the Pod than the version which was responsible for triggering the update. + indexer, informer := cache.NewIndexerInformer(podListWatcher, &v1.Pod{}, 0, cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + if key, err := cache.MetaNamespaceKeyFunc(obj); err == nil { + queue.Add(key) + } + }, + UpdateFunc: func(old interface{}, new interface{}) { + if key, err := cache.MetaNamespaceKeyFunc(new); err == nil { + queue.Add(key) + } + }, + DeleteFunc: func(obj interface{}) { + // IndexerInformer uses a delta queue, therefore for deletes we have to use this key function. + if key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj); err == nil { + queue.Add(key) + } + }, + }, cache.Indexers{}) + evicter := newPodEvicter(clientset.PolicyV1beta1(), *terminationGracePeriodSeconds) + controller := NewController(queue, indexer, informer, evicter, *taintedIncubationPeriodSeconds) + + stop := make(chan struct{}) + defer close(stop) + go controller.Run(1, stop) + + // todo: watch sigterm + // todo: recover panic to log + select {} +} + +type podEvicter struct { + client v1beta1.PolicyV1beta1Interface + defaultDeleteOptions *metav1.DeleteOptions +} + +func newPodEvicter(client v1beta1.PolicyV1beta1Interface, gracePeriodSeconds int64) *podEvicter { + return &podEvicter{client: client, defaultDeleteOptions: &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriodSeconds}} +} + +func (p *podEvicter) Evict(namespace, podName string) error { + return p.client.Evictions(namespace).Evict(newEviction(namespace, podName, p.defaultDeleteOptions)) +} + +func newEviction(ns, evictionName string, deleteOption *metav1.DeleteOptions) *policy.Eviction { + return &policy.Eviction{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "Policy/v1beta1", + Kind: "Eviction", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: evictionName, + Namespace: ns, + }, + DeleteOptions: deleteOption, + } +} diff --git a/go.mod b/go.mod index 9629bcf..63693c2 100644 --- a/go.mod +++ b/go.mod @@ -8,12 +8,14 @@ require ( github.com/gorilla/mux v1.7.0 github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/opencontainers/go-digest v1.0.0-rc1 + github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.3.0 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect k8s.io/api v0.17.2 k8s.io/apimachinery v0.17.2 k8s.io/client-go v0.17.2 //v11.0.0+incompatible + k8s.io/klog v1.0.0 k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab sigs.k8s.io/yaml v1.1.0 ) diff --git a/go.sum b/go.sum index b7e4250..a9fb17a 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,7 @@ github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJA github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -52,6 +53,7 @@ github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= @@ -61,8 +63,10 @@ github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U= github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= @@ -96,6 +100,8 @@ github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -108,7 +114,6 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= @@ -116,7 +121,6 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 h1:7KByu05hhLed2MO29w7p1XfZvZ13m8mub3shuVftRs0= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -153,7 +157,6 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -170,13 +173,11 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 h1:ZUjXAXmrAyrmmCPHgCA/vChHcpsX27MZ3yBonD/z1KE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -187,7 +188,6 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 16915c1ceb4750b7f86f61f0640580aa29e16e03 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Thu, 23 Jan 2020 12:42:22 +0100 Subject: [PATCH 02/11] Submit k8s event for evict --- cmd/evicter/controller.go | 31 ++++++++++++++++++++----------- cmd/evicter/main.go | 38 +++++++++++++++++++++++++++----------- go.sum | 7 +++++++ 3 files changed, 54 insertions(+), 22 deletions(-) diff --git a/cmd/evicter/controller.go b/cmd/evicter/controller.go index 2f3e073..f61519e 100644 --- a/cmd/evicter/controller.go +++ b/cmd/evicter/controller.go @@ -15,7 +15,7 @@ import ( ) type podProvisioner interface { - Evict(namespace, podName string) error + Evict(pod *v1.Pod, reason string) error } type Controller struct { @@ -49,6 +49,13 @@ func (c *Controller) processNextItem() bool { return true } +const ( + annotationPreventEviction = "k-rails/tainted-prevent-eviction" + annotationTimestamp = "k-rails/tainted-timestamp" + annotationReason = "k-rails/tainted-reason" +) +const defaultEvictionReason = "exec" + // evictPod is the business logic of the controller. it checks the the eviction rules and conditions before calling the pod provisioner. func (c *Controller) evictPod(key string) error { obj, exists, err := c.podStore.GetByKey(key) @@ -58,24 +65,26 @@ func (c *Controller) evictPod(key string) error { case !exists: return nil } - pod := obj.(*v1.Pod) + pod, ok := obj.(*v1.Pod) + if !ok { + return fmt.Errorf("unsupported type: %T", obj) + } if !canEvict(pod, c.incubationPeriodSeconds) { return nil } - ns, name, err := cache.SplitMetaNamespaceKey(key) - if err != nil { - return err + reason, ok := pod.Annotations[annotationReason] + if !ok || reason == "" { + reason = defaultEvictionReason } - return c.podProvisioner.Evict(ns, name) -} -const ( - annotationPreventEviction = "k-rails/tainted-prevent-eviction" - annotationTimestamp = "k-rails/tainted-timestamp" -) + return c.podProvisioner.Evict(pod, reason) +} func canEvict(pod *v1.Pod, incubationPeriod time.Duration) bool { + if pod == nil { + return false + } val, ok := pod.Annotations[annotationPreventEviction] if ok { if val == "yes" || val == "true" { diff --git a/cmd/evicter/main.go b/cmd/evicter/main.go index d859ae3..04eaaf9 100644 --- a/cmd/evicter/main.go +++ b/cmd/evicter/main.go @@ -3,13 +3,16 @@ package main import ( "flag" + "github.com/pkg/errors" v1 "k8s.io/api/core/v1" policy "k8s.io/api/policy/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/kubernetes/typed/policy/v1beta1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/events" "k8s.io/client-go/util/workqueue" "k8s.io/klog" ) @@ -62,11 +65,17 @@ func main() { } }, }, cache.Indexers{}) - evicter := newPodEvicter(clientset.PolicyV1beta1(), *terminationGracePeriodSeconds) - controller := NewController(queue, indexer, informer, evicter, *taintedIncubationPeriodSeconds) stop := make(chan struct{}) defer close(stop) + + eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: clientset.EventsV1beta1().Events("")}) + eventBroadcaster.StartRecordingToSink(stop) + defer eventBroadcaster.Shutdown() + + evicter := newPodEvicter(clientset.PolicyV1beta1(), eventBroadcaster.NewRecorder(scheme.Scheme, "k-rail-evicter"), *terminationGracePeriodSeconds) + controller := NewController(queue, indexer, informer, evicter, *taintedIncubationPeriodSeconds) + go controller.Run(1, stop) // todo: watch sigterm @@ -76,27 +85,34 @@ func main() { type podEvicter struct { client v1beta1.PolicyV1beta1Interface + eventRecorder events.EventRecorder defaultDeleteOptions *metav1.DeleteOptions } -func newPodEvicter(client v1beta1.PolicyV1beta1Interface, gracePeriodSeconds int64) *podEvicter { - return &podEvicter{client: client, defaultDeleteOptions: &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriodSeconds}} +func newPodEvicter(client v1beta1.PolicyV1beta1Interface, recorder events.EventRecorder, gracePeriodSeconds int64) *podEvicter { + return &podEvicter{ + client: client, + eventRecorder: recorder, + defaultDeleteOptions: &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriodSeconds}, + } } -func (p *podEvicter) Evict(namespace, podName string) error { - return p.client.Evictions(namespace).Evict(newEviction(namespace, podName, p.defaultDeleteOptions)) +func (p *podEvicter) Evict(pod *v1.Pod, reason string) error { + err := p.client.Evictions(pod.Namespace).Evict(newEviction(pod, p.defaultDeleteOptions)) + if err != nil { + return errors.Wrap(err, "eviction") + } + p.eventRecorder.Eventf(pod, nil, v1.EventTypeNormal, reason, "Eviction", "") + return nil } -func newEviction(ns, evictionName string, deleteOption *metav1.DeleteOptions) *policy.Eviction { +func newEviction(pod *v1.Pod, deleteOption *metav1.DeleteOptions) *policy.Eviction { return &policy.Eviction{ TypeMeta: metav1.TypeMeta{ APIVersion: "Policy/v1beta1", Kind: "Eviction", }, - ObjectMeta: metav1.ObjectMeta{ - Name: evictionName, - Namespace: ns, - }, + ObjectMeta: pod.ObjectMeta, DeleteOptions: deleteOption, } } diff --git a/go.sum b/go.sum index a9fb17a..88f5ae5 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,7 @@ github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZ github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= @@ -65,6 +66,7 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -94,8 +96,10 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -183,9 +187,11 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -204,6 +210,7 @@ k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUc k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab h1:I3f2hcBrepGRXI1z4sukzAb8w1R4eqbsHrAsx06LGYM= From e08a4dd31b25e0ee92a66c32632943caf013c369 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Wed, 29 Jan 2020 17:38:12 +0100 Subject: [PATCH 03/11] Add evicter to docker image --- .gitignore | 1 + Dockerfile | 3 ++- Makefile | 24 +++++++++++++++++++----- cmd/{ => k-rail}/main.go | 0 4 files changed, 22 insertions(+), 6 deletions(-) rename cmd/{ => k-rail}/main.go (100%) diff --git a/.gitignore b/.gitignore index 6b612cb..8e3620c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /k-rail +/evicter vendor diff --git a/Dockerfile b/Dockerfile index 1a69d35..e5e859f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ ARG GO_VERSION=1.13 FROM golang:${GO_VERSION}-buster AS builder WORKDIR /build COPY ./ /build/ -RUN make test +RUN make clean test RUN make build # Production image build stage @@ -11,5 +11,6 @@ FROM scratch EXPOSE 8443/tcp COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt COPY --from=builder /build/k-rail /k-rail +COPY --from=builder /build/evicter /evicter USER 65534 ENTRYPOINT ["/k-rail", "-config", "/config/config.yml"] diff --git a/Makefile b/Makefile index d5d5857..beec44a 100644 --- a/Makefile +++ b/Makefile @@ -9,13 +9,27 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +.PHONY: all build test image clean -ensure: - dep ensure +LDFLAGS = -extldflags=-static -s -w +BUILD_FLAGS = -mod=readonly -ldflags '$(LDFLAGS)' -trimpath +BUILD_VERSION ?= manual +IMAGE_NAME = "cruise/k-rail:${BUILD_VERSION}" + +all: dist + +dist: image + +clean: + rm -f evicter k-rail + go mod verify build: - GO111MODULE=on CGO_ENABLED=0 go build -o k-rail cmd/main.go + GO111MODULE=on CGO_ENABLED=0 go build ${BUILD_FLAGS} -o k-rail cmd/k-rail/main.go + GO111MODULE=on CGO_ENABLED=0 go build ${BUILD_FLAGS} -o evicter cmd/evicter/*.go test: - GO111MODULE=on CGO_ENABLED=1 go test -race -cover $(shell go list ./... | grep -v /vendor/) - \ No newline at end of file + GO111MODULE=on CGO_ENABLED=1 go test -mod=readonly -race ./... + +image: build + docker build --pull -t $(IMAGE_NAME) . \ No newline at end of file diff --git a/cmd/main.go b/cmd/k-rail/main.go similarity index 100% rename from cmd/main.go rename to cmd/k-rail/main.go From 5b1f37d79281a14c1fb03876f4d89c236b5f9f42 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Fri, 31 Jan 2020 13:56:23 +0100 Subject: [PATCH 04/11] Format Makefile --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index beec44a..dc9f74b 100644 --- a/Makefile +++ b/Makefile @@ -25,11 +25,11 @@ clean: go mod verify build: - GO111MODULE=on CGO_ENABLED=0 go build ${BUILD_FLAGS} -o k-rail cmd/k-rail/main.go - GO111MODULE=on CGO_ENABLED=0 go build ${BUILD_FLAGS} -o evicter cmd/evicter/*.go + GO111MODULE=on CGO_ENABLED=0 go build ${BUILD_FLAGS} -o k-rail cmd/k-rail/main.go + GO111MODULE=on CGO_ENABLED=0 go build ${BUILD_FLAGS} -o evicter cmd/evicter/*.go test: - GO111MODULE=on CGO_ENABLED=1 go test -mod=readonly -race ./... + GO111MODULE=on CGO_ENABLED=1 go test -mod=readonly -race ./... image: build docker build --pull -t $(IMAGE_NAME) . \ No newline at end of file From 5014cde3aba8775cd75da740042fafdf9e3b9845 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Fri, 31 Jan 2020 13:57:29 +0100 Subject: [PATCH 05/11] Fix annotation prefix --- cmd/evicter/README.md | 5 +++-- cmd/evicter/controller.go | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/evicter/README.md b/cmd/evicter/README.md index b5ad1d4..bdedf24 100644 --- a/cmd/evicter/README.md +++ b/cmd/evicter/README.md @@ -1,5 +1,6 @@ # Evict tainted pods after period Operator that runs within k8s to find and evict tainted pods -* `k-rails/tainted-timestamp` -* `k-rails/tainted-prevent-eviction` +* `k-rail/tainted-timestamp` +* `k-rail/tainted-prevent-eviction` +* `k-rail/reason` diff --git a/cmd/evicter/controller.go b/cmd/evicter/controller.go index f61519e..397bff6 100644 --- a/cmd/evicter/controller.go +++ b/cmd/evicter/controller.go @@ -50,9 +50,9 @@ func (c *Controller) processNextItem() bool { } const ( - annotationPreventEviction = "k-rails/tainted-prevent-eviction" - annotationTimestamp = "k-rails/tainted-timestamp" - annotationReason = "k-rails/tainted-reason" + annotationPreventEviction = "k-rail/tainted-prevent-eviction" + annotationTimestamp = "k-rail/tainted-timestamp" + annotationReason = "k-rail/tainted-reason" ) const defaultEvictionReason = "exec" From 1d75425ec317b9170ca1ee1fe65cb3e453b43cb3 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Mon, 3 Feb 2020 13:36:04 +0100 Subject: [PATCH 06/11] Add leader lock to evicter --- cmd/evicter/README.md | 8 +-- cmd/evicter/controller.go | 20 +++--- cmd/evicter/main.go | 147 +++++++++++++++++++++++++++----------- go.mod | 2 + go.sum | 2 + 5 files changed, 125 insertions(+), 54 deletions(-) diff --git a/cmd/evicter/README.md b/cmd/evicter/README.md index bdedf24..30d6a76 100644 --- a/cmd/evicter/README.md +++ b/cmd/evicter/README.md @@ -1,6 +1,6 @@ # Evict tainted pods after period -Operator that runs within k8s to find and evict tainted pods +Operator that runs within k8s to find and evict `tainted` pods. -* `k-rail/tainted-timestamp` -* `k-rail/tainted-prevent-eviction` -* `k-rail/reason` +* `k-rail/tainted-timestamp` store the unix timestamp when the root event happend +* `k-rail/tainted-prevent-eviction` is a break-glass annotation to prevent automated eviction +* `k-rail/reason` intended for humans diff --git a/cmd/evicter/controller.go b/cmd/evicter/controller.go index 397bff6..e44334a 100644 --- a/cmd/evicter/controller.go +++ b/cmd/evicter/controller.go @@ -15,7 +15,7 @@ import ( ) type podProvisioner interface { - Evict(pod *v1.Pod, reason string) error + Evict(pod *v1.Pod, reason, msg string) error } type Controller struct { @@ -50,11 +50,16 @@ func (c *Controller) processNextItem() bool { } const ( + // annotationPreventEviction is a break-glass annotation to prevent automated eviction annotationPreventEviction = "k-rail/tainted-prevent-eviction" + // annotationTimestamp stores the unix timestamp when the root event happened annotationTimestamp = "k-rail/tainted-timestamp" + // annotationReason is used to define any additional reason in a human readable form annotationReason = "k-rail/tainted-reason" ) -const defaultEvictionReason = "exec" + +const defaultEvictionReason = "Tainted" +const noEvictionNote = "Pod was marked as tainted" // evictPod is the business logic of the controller. it checks the the eviction rules and conditions before calling the pod provisioner. func (c *Controller) evictPod(key string) error { @@ -73,12 +78,11 @@ func (c *Controller) evictPod(key string) error { return nil } - reason, ok := pod.Annotations[annotationReason] - if !ok || reason == "" { - reason = defaultEvictionReason + msg, ok := pod.Annotations[annotationReason] + if !ok || msg == "" { + msg = noEvictionNote } - - return c.podProvisioner.Evict(pod, reason) + return c.podProvisioner.Evict(pod, defaultEvictionReason, msg) } func canEvict(pod *v1.Pod, incubationPeriod time.Duration) bool { @@ -138,7 +142,7 @@ func (c *Controller) handleErr(err error, key interface{}) { const reconciliationTick = 30 * time.Second const startupGracePeriod = 90 * time.Second -func (c *Controller) Run(threadiness int, stopCh chan struct{}) { +func (c *Controller) Run(threadiness int, stopCh <-chan struct{}) { defer runtime.HandleCrash() // Let the workers stop when we are done diff --git a/cmd/evicter/main.go b/cmd/evicter/main.go index 04eaaf9..2f55c02 100644 --- a/cmd/evicter/main.go +++ b/cmd/evicter/main.go @@ -1,22 +1,34 @@ package main import ( + "context" "flag" + "os" + "os/signal" + "syscall" + "time" + "github.com/cruise-automation/k-rail/resource" + "github.com/golang/glog" + "github.com/google/uuid" "github.com/pkg/errors" v1 "k8s.io/api/core/v1" policy "k8s.io/api/policy/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/kubernetes/typed/policy/v1beta1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/tools/events" + "k8s.io/client-go/tools/leaderelection" + "k8s.io/client-go/tools/leaderelection/resourcelock" + "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" "k8s.io/klog" ) + func main() { var ( kubeconfig = flag.String("kubeconfig", "", "absolute path to the kubeconfig file: `/.kube/config`") @@ -24,6 +36,9 @@ func main() { labelSelector = flag.String("label-selector", "tainted=true", "label selector to discover tainted pods") terminationGracePeriodSeconds = flag.Int64("termination-grace-period", 30, "pod termination grace period in seconds") taintedIncubationPeriodSeconds = flag.Int64("incubation-period", 24*60*60, "time in seconds a tainted pod can run before eviction") + instanceID = flag.String("instance", uuid.New().String(), "the unique holder identity. used for leader lock") + leaseLockName = flag.String("lease-lock-name", "k-rail-evicter-lock", "the lease lock resource name") + leaseLockNamespace = flag.String("lease-lock-namespace", "k-rail", "the lease lock resource namespace") ) flag.Parse() flag.Set("logtostderr", "true") // glog: no disk log @@ -37,59 +52,105 @@ func main() { if err != nil { klog.Fatal(err) } - podListWatcher := cache.NewFilteredListWatchFromClient(clientset.CoreV1().RESTClient(), "pods", metav1.NamespaceDefault, - func(options *metav1.ListOptions) { - options.LabelSelector = *labelSelector - }) - - queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) - // Bind the workqueue to a cache with the help of an informer. This way we make sure that - // whenever the cache is updated, the pod key is added to the workqueue. - // Note that when we finally process the item from the workqueue, we might see a newer version - // of the Pod than the version which was responsible for triggering the update. - indexer, informer := cache.NewIndexerInformer(podListWatcher, &v1.Pod{}, 0, cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - if key, err := cache.MetaNamespaceKeyFunc(obj); err == nil { - queue.Add(key) - } - }, - UpdateFunc: func(old interface{}, new interface{}) { - if key, err := cache.MetaNamespaceKeyFunc(new); err == nil { - queue.Add(key) - } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + watchSigTerm(cancel) + + br := record.NewBroadcaster() + br.StartRecordingToSink(&corev1.EventSinkImpl{Interface: clientset.CoreV1().Events(metav1.NamespaceAll)}) + br.StartLogging(glog.Infof) + defer br.Shutdown() + eventRecorder := br.NewRecorder(scheme.Scheme, v1.EventSource{Component: "k-rail-evicter"}) + + lock := &resourcelock.LeaseLock{ + LeaseMeta: metav1.ObjectMeta{ + Name: *leaseLockName, + Namespace: *leaseLockNamespace, }, - DeleteFunc: func(obj interface{}) { - // IndexerInformer uses a delta queue, therefore for deletes we have to use this key function. - if key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj); err == nil { - queue.Add(key) - } + Client: clientset.CoordinationV1(), + LockConfig: resourcelock.ResourceLockConfig{ + Identity: *instanceID, + EventRecorder: eventRecorder, }, - }, cache.Indexers{}) + } - stop := make(chan struct{}) - defer close(stop) + // start the leader election code loop + leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{ + Lock: lock, + ReleaseOnCancel: true, + LeaseDuration: 60 * time.Second, + RenewDeadline: 15 * time.Second, + RetryPeriod: 5 * time.Second, + Callbacks: leaderelection.LeaderCallbacks{ + OnStartedLeading: func(ctx context.Context) { + klog.Infof("Starting leader: %s", *instanceID) + podListWatcher := cache.NewFilteredListWatchFromClient(clientset.CoreV1().RESTClient(), "pods", metav1.NamespaceAll, + func(options *metav1.ListOptions) { + options.LabelSelector = *labelSelector + }) - eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: clientset.EventsV1beta1().Events("")}) - eventBroadcaster.StartRecordingToSink(stop) - defer eventBroadcaster.Shutdown() + queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) - evicter := newPodEvicter(clientset.PolicyV1beta1(), eventBroadcaster.NewRecorder(scheme.Scheme, "k-rail-evicter"), *terminationGracePeriodSeconds) - controller := NewController(queue, indexer, informer, evicter, *taintedIncubationPeriodSeconds) + // Bind the workqueue to a cache with the help of an informer. This way we make sure that + // whenever the cache is updated, the pod key is added to the workqueue. + // Note that when we finally process the item from the workqueue, we might see a newer version + // of the Pod than the version which was responsible for triggering the update. + indexer, informer := cache.NewIndexerInformer(podListWatcher, &v1.Pod{}, 0, cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + if key, err := cache.MetaNamespaceKeyFunc(obj); err == nil { + queue.Add(key) + } + }, + UpdateFunc: func(old interface{}, new interface{}) { + if key, err := cache.MetaNamespaceKeyFunc(new); err == nil { + queue.Add(key) + } + }, + DeleteFunc: func(obj interface{}) { + // IndexerInformer uses a delta queue, therefore for deletes we have to use this key function. + if key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj); err == nil { + queue.Add(key) + } + }, + }, cache.Indexers{}) - go controller.Run(1, stop) + evicter := newPodEvicter(clientset.PolicyV1beta1(), eventRecorder, *terminationGracePeriodSeconds) + controller := NewController(queue, indexer, informer, evicter, *taintedIncubationPeriodSeconds) + controller.Run(1, ctx.Done()) + }, + OnStoppedLeading: func() { + // we can do cleanup here + klog.Infof("Leader lost: %s", *instanceID) + os.Exit(0) + }, + OnNewLeader: func(identity string) { + if identity == *instanceID { + return + } + klog.Infof("New leader elected: %s", identity) + }, + }, + }) +} - // todo: watch sigterm - // todo: recover panic to log - select {} +func watchSigTerm(cancel context.CancelFunc) { + ch := make(chan os.Signal, 1) + signal.Notify(ch, os.Interrupt, syscall.SIGTERM) + go func() { + <-ch + klog.Info("Received termination, signaling shutdown") + cancel() + }() } type podEvicter struct { client v1beta1.PolicyV1beta1Interface - eventRecorder events.EventRecorder + eventRecorder record.EventRecorder defaultDeleteOptions *metav1.DeleteOptions } -func newPodEvicter(client v1beta1.PolicyV1beta1Interface, recorder events.EventRecorder, gracePeriodSeconds int64) *podEvicter { +func newPodEvicter(client v1beta1.PolicyV1beta1Interface, recorder record.EventRecorder, gracePeriodSeconds int64) *podEvicter { return &podEvicter{ client: client, eventRecorder: recorder, @@ -97,12 +158,14 @@ func newPodEvicter(client v1beta1.PolicyV1beta1Interface, recorder events.EventR } } -func (p *podEvicter) Evict(pod *v1.Pod, reason string) error { +// Evict calls the k8s api to evict the given pod. Reason and notes are stored with the audit event. +func (p *podEvicter) Evict(pod *v1.Pod, reason, msg string) error { err := p.client.Evictions(pod.Namespace).Evict(newEviction(pod, p.defaultDeleteOptions)) + klog.Infof("Evicted pod %q (UID: %s)", resource.GetResourceName(pod.ObjectMeta), pod.UID) if err != nil { return errors.Wrap(err, "eviction") } - p.eventRecorder.Eventf(pod, nil, v1.EventTypeNormal, reason, "Eviction", "") + p.eventRecorder.Eventf(pod, v1.EventTypeNormal, reason, msg) return nil } diff --git a/go.mod b/go.mod index 63693c2..5932301 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,9 @@ go 1.12 require ( github.com/gobwas/glob v0.2.3 + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/protobuf v1.3.4 // indirect + github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.7.0 github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/opencontainers/go-digest v1.0.0-rc1 diff --git a/go.sum b/go.sum index 88f5ae5..16f540a 100644 --- a/go.sum +++ b/go.sum @@ -21,6 +21,7 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -34,6 +35,7 @@ github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= From 7bc0b3483e7dc3106ca0a4acd8645cecfc44cc5f Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Mon, 3 Feb 2020 13:41:44 +0100 Subject: [PATCH 07/11] Add evicter to helm chart (WIP) --- charts/k-rail/templates/deployment.yaml | 117 ++++++++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/charts/k-rail/templates/deployment.yaml b/charts/k-rail/templates/deployment.yaml index 5ef4252..a7ba575 100644 --- a/charts/k-rail/templates/deployment.yaml +++ b/charts/k-rail/templates/deployment.yaml @@ -214,3 +214,120 @@ kind: ServiceAccount metadata: name: k-rail namespace: {{ .Release.Namespace }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: k-rail-evicter + namespace: {{ .Release.Namespace }} + labels: + name: k-rail-evicter +spec: + replicas: 2 # 2 required for leader lock testing + selector: + matchLabels: + name: k-rail-evicter + template: + metadata: + name: k-rail-evicter + labels: + name: k-rail-evicter + spec: + serviceAccountName: k-rail-evicter + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + containers: + - name: k-rail + command: ["/evicter"] + # image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + image: "alpetest/k-rail-wip:manual" + # imagePullPolicy: {{ .Values.image.pullPolicy }} + imagePullPolicy: Always + resources: + {{- toYaml .Values.resources | nindent 12 }} + securityContext: + readOnlyRootFilesystem: true +# {{- with .Values.nodeSelector }} +# nodeSelector: +# {{- toYaml . | nindent 8 }} +# {{- end }} +# {{- with .Values.affinity }} +# affinity: +# {{- toYaml . | nindent 8 }} +# {{- end }} +# {{- with .Values.tolerations }} +# tolerations: +# {{- toYaml . | nindent 8 }} +# {{- end }} +--- +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: k-rail-evicter + namespace: {{ .Release.Namespace }} +spec: + minAvailable: 1 + selector: + matchLabels: + name: k-rail-evicter +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: k-rail-evicter + namespace: {{ .Release.Namespace }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: k-rail-evicter +rules: + - apiGroups: + - "" # "" indicates the core API group + resources: + - pods + verbs: + - get + - list + - watch + - apiGroups: + - "" # "" indicates the core API group + resources: + - pods/eviction + verbs: + - create + - apiGroups: + - "" # "" indicates the core API group + resources: + - events + verbs: + - create + - patch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - patch + - update + +# todo: anything to do about jobs and daemon-sets? +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: k-rail-evicter +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: k-rail-evicter +subjects: + - kind: ServiceAccount + name: k-rail-evicter + namespace: {{ .Release.Namespace }} \ No newline at end of file From 486f884341725e04a583ce538e35c1fc805fd40b Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Tue, 11 Feb 2020 17:31:05 +0100 Subject: [PATCH 08/11] Tests and minor refactorings --- cmd/evicter/controller.go | 18 ++-- cmd/evicter/controller_test.go | 167 +++++++++++++++++++++++++++++++++ cmd/evicter/main.go | 3 +- 3 files changed, 176 insertions(+), 12 deletions(-) create mode 100644 cmd/evicter/controller_test.go diff --git a/cmd/evicter/controller.go b/cmd/evicter/controller.go index e44334a..9898a21 100644 --- a/cmd/evicter/controller.go +++ b/cmd/evicter/controller.go @@ -19,7 +19,7 @@ type podProvisioner interface { } type Controller struct { - podStore cache.Indexer + podStore cache.Store queue workqueue.RateLimitingInterface informer cache.Controller podProvisioner podProvisioner @@ -27,7 +27,7 @@ type Controller struct { started time.Time } -func NewController(queue workqueue.RateLimitingInterface, indexer cache.Indexer, informer cache.Controller, podProvisioner podProvisioner, incubationPeriodSeconds int64) *Controller { +func NewController(queue workqueue.RateLimitingInterface, indexer cache.Store, informer cache.Controller, podProvisioner podProvisioner, incubationPeriodSeconds int64) *Controller { return &Controller{ informer: informer, podStore: indexer, @@ -53,9 +53,9 @@ const ( // annotationPreventEviction is a break-glass annotation to prevent automated eviction annotationPreventEviction = "k-rail/tainted-prevent-eviction" // annotationTimestamp stores the unix timestamp when the root event happened - annotationTimestamp = "k-rail/tainted-timestamp" + annotationTimestamp = "k-rail/tainted-timestamp" // annotationReason is used to define any additional reason in a human readable form - annotationReason = "k-rail/tainted-reason" + annotationReason = "k-rail/tainted-reason" ) const defaultEvictionReason = "Tainted" @@ -89,14 +89,12 @@ func canEvict(pod *v1.Pod, incubationPeriod time.Duration) bool { if pod == nil { return false } - val, ok := pod.Annotations[annotationPreventEviction] - if ok { - if val == "yes" || val == "true" { - return false - } + switch pod.Annotations[annotationPreventEviction] { + case "yes", "true", "1", "YES", "TRUE", "Yes", "True": + return false } - val, ok = pod.Annotations[annotationTimestamp] + val, ok := pod.Annotations[annotationTimestamp] if ok { i, err := strconv.ParseInt(val, 10, 64) if err != nil { diff --git a/cmd/evicter/controller_test.go b/cmd/evicter/controller_test.go new file mode 100644 index 0000000..deaceb1 --- /dev/null +++ b/cmd/evicter/controller_test.go @@ -0,0 +1,167 @@ +package main + +import ( + "reflect" + "strconv" + "testing" + "time" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/cache" +) + +func TestCanEvict(t *testing.T) { + now := int(time.Now().Unix()) + specs := map[string]struct { + srcAnn map[string]string + expResult bool + }{ + "with timestamp after incubation period": { + srcAnn: map[string]string{ + "k-rail/tainted-timestamp": strconv.Itoa(now - 1), + "k-rail/tainted-reason": "test", + }, + expResult: true, + }, + "with timestamp in incubation period": { + srcAnn: map[string]string{ + "k-rail/tainted-timestamp": strconv.Itoa(now), + "k-rail/tainted-reason": "test", + }, + expResult: false, + }, + "without timestamp annotation": { + srcAnn: map[string]string{ + "k-rail/tainted-reason": "test", + }, + expResult: true, + }, + "with timestamp containing non timestamp string": { + srcAnn: map[string]string{ + "k-rail/tainted-timestamp": "", + "k-rail/tainted-reason": "test", + }, + expResult: true, + }, + "with preventEviction annotation": { + srcAnn: map[string]string{ + "k-rail/tainted-timestamp": strconv.Itoa(now - 1), + "k-rail/tainted-reason": "test", + "k-rail/tainted-prevent-eviction": "true", + }, + expResult: false, + }, + "with preventEviction annotation - uppercase": { + srcAnn: map[string]string{ + "k-rail/tainted-timestamp": strconv.Itoa(now - 1), + "k-rail/tainted-reason": "test", + "k-rail/tainted-prevent-eviction": "TRUE", + }, + expResult: false, + }, + "with preventEviction annotation - yes": { + srcAnn: map[string]string{ + "k-rail/tainted-timestamp": strconv.Itoa(now - 1), + "k-rail/tainted-reason": "test", + "k-rail/tainted-prevent-eviction": "yes", + }, + expResult: false, + }, + "with preventEviction annotation - non bool": { + srcAnn: map[string]string{ + "k-rail/tainted-timestamp": strconv.Itoa(now - 1), + "k-rail/tainted-reason": "test", + "k-rail/tainted-prevent-eviction": "", + }, + expResult: true, + }, + } + for msg, spec := range specs { + t.Run(msg, func(t *testing.T) { + pod := v1.Pod{ObjectMeta: metav1.ObjectMeta{Annotations: spec.srcAnn}} + if got := canEvict(&pod, time.Second); spec.expResult != got { + t.Errorf("expected %v but got %v", spec.expResult, got) + } + }) + } +} + +func TestEvictPod(t *testing.T) { + now := int(time.Now().Unix()) + + specs := map[string]struct { + srcAnn map[string]string + expReason string + expMsg string + expNoEviction bool + }{ + "evicted with custom reason": { + srcAnn: map[string]string{ + "k-rail/tainted-timestamp": strconv.Itoa(now - 1), + "k-rail/tainted-reason": "test", + }, + expReason: "Tainted", + expMsg: "test", + }, + "evicted with default reason": { + srcAnn: map[string]string{ + "k-rail/tainted-timestamp": strconv.Itoa(now - 1), + }, + expReason: "Tainted", + expMsg: noEvictionNote, + }, + "not evicted with annotation": { + srcAnn: map[string]string{ + "k-rail/tainted-timestamp": strconv.Itoa(now - 1), + "k-rail/tainted-prevent-eviction": "yes", + }, + expNoEviction: true, + }, + } + for msg, spec := range specs { + t.Run(msg, func(t *testing.T) { + store := cache.NewStore(cache.MetaNamespaceKeyFunc) + pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "myPod", Annotations: spec.srcAnn}} + store.Add(pod) + prov := &recordingPodProvisioner{} + c := NewController(nil, store, nil, prov, 1) + // when + err := c.evictPod("myPod") + // then + if err != nil { + t.Fatalf("unexpected error: %+v", err) + } + if spec.expNoEviction { + if prov.evictedPods != nil { + t.Fatalf("expected no call but got %v", prov.evictedPods) + } + return + } + // there should be 1 call + if exp, got := []*v1.Pod{pod}, prov.evictedPods; !reflect.DeepEqual(exp, got) { + t.Errorf("expected %v but got %v", exp, got) + } + if exp, got := []string{spec.expReason}, prov.reasons; !reflect.DeepEqual(exp, got) { + t.Errorf("expected %v but got %v", exp, got) + } + if exp, got := []string{spec.expMsg}, prov.msgs; !reflect.DeepEqual(exp, got) { + t.Errorf("expected %v but got %v", exp, got) + } + }) + } +} + +type recordingPodProvisioner struct { + evictedPods []*v1.Pod + reasons []string + msgs []string + result error +} + +func (r *recordingPodProvisioner) Evict(pod *v1.Pod, reason, msg string) error { + r.evictedPods = append(r.evictedPods, pod) + r.reasons = append(r.reasons, reason) + r.msgs = append(r.msgs, msg) + return r.result +} diff --git a/cmd/evicter/main.go b/cmd/evicter/main.go index 2f55c02..747bbed 100644 --- a/cmd/evicter/main.go +++ b/cmd/evicter/main.go @@ -28,7 +28,6 @@ import ( "k8s.io/klog" ) - func main() { var ( kubeconfig = flag.String("kubeconfig", "", "absolute path to the kubeconfig file: `/.kube/config`") @@ -165,7 +164,7 @@ func (p *podEvicter) Evict(pod *v1.Pod, reason, msg string) error { if err != nil { return errors.Wrap(err, "eviction") } - p.eventRecorder.Eventf(pod, v1.EventTypeNormal, reason, msg) + p.eventRecorder.Eventf(pod, v1.EventTypeNormal, reason, msg) return nil } From 5e08f881cc10f192ded59bc0eaff621791f1495a Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Thu, 6 Aug 2020 17:17:28 +0200 Subject: [PATCH 09/11] Revert "Add evicter to helm chart (WIP)" This reverts commit 7bc0b3483e7dc3106ca0a4acd8645cecfc44cc5f. --- charts/k-rail/templates/deployment.yaml | 117 ------------------------ 1 file changed, 117 deletions(-) diff --git a/charts/k-rail/templates/deployment.yaml b/charts/k-rail/templates/deployment.yaml index a7ba575..5ef4252 100644 --- a/charts/k-rail/templates/deployment.yaml +++ b/charts/k-rail/templates/deployment.yaml @@ -214,120 +214,3 @@ kind: ServiceAccount metadata: name: k-rail namespace: {{ .Release.Namespace }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: k-rail-evicter - namespace: {{ .Release.Namespace }} - labels: - name: k-rail-evicter -spec: - replicas: 2 # 2 required for leader lock testing - selector: - matchLabels: - name: k-rail-evicter - template: - metadata: - name: k-rail-evicter - labels: - name: k-rail-evicter - spec: - serviceAccountName: k-rail-evicter - securityContext: - runAsNonRoot: true - runAsUser: 1000 - fsGroup: 2000 - containers: - - name: k-rail - command: ["/evicter"] - # image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - image: "alpetest/k-rail-wip:manual" - # imagePullPolicy: {{ .Values.image.pullPolicy }} - imagePullPolicy: Always - resources: - {{- toYaml .Values.resources | nindent 12 }} - securityContext: - readOnlyRootFilesystem: true -# {{- with .Values.nodeSelector }} -# nodeSelector: -# {{- toYaml . | nindent 8 }} -# {{- end }} -# {{- with .Values.affinity }} -# affinity: -# {{- toYaml . | nindent 8 }} -# {{- end }} -# {{- with .Values.tolerations }} -# tolerations: -# {{- toYaml . | nindent 8 }} -# {{- end }} ---- -apiVersion: policy/v1beta1 -kind: PodDisruptionBudget -metadata: - name: k-rail-evicter - namespace: {{ .Release.Namespace }} -spec: - minAvailable: 1 - selector: - matchLabels: - name: k-rail-evicter ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: k-rail-evicter - namespace: {{ .Release.Namespace }} ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: k-rail-evicter -rules: - - apiGroups: - - "" # "" indicates the core API group - resources: - - pods - verbs: - - get - - list - - watch - - apiGroups: - - "" # "" indicates the core API group - resources: - - pods/eviction - verbs: - - create - - apiGroups: - - "" # "" indicates the core API group - resources: - - events - verbs: - - create - - patch - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - patch - - update - -# todo: anything to do about jobs and daemon-sets? ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: k-rail-evicter -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: k-rail-evicter -subjects: - - kind: ServiceAccount - name: k-rail-evicter - namespace: {{ .Release.Namespace }} \ No newline at end of file From ea30a1b105f430458dd24b89cfed7b11ccfe170c Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Thu, 6 Aug 2020 17:22:30 +0200 Subject: [PATCH 10/11] Cleanup go.mod --- go.mod | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.mod b/go.mod index 5932301..0cc3cf8 100644 --- a/go.mod +++ b/go.mod @@ -21,5 +21,3 @@ require ( k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab sigs.k8s.io/yaml v1.1.0 ) - -replace git.apache.org/thrift.git => github.com/apache/thrift v0.12.0 From d4875f7becf7a9c19a76369e4120449731270dd1 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Thu, 6 Aug 2020 18:08:46 +0200 Subject: [PATCH 11/11] Add probe endpoint --- cmd/evicter/main.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/cmd/evicter/main.go b/cmd/evicter/main.go index 747bbed..b3f370e 100644 --- a/cmd/evicter/main.go +++ b/cmd/evicter/main.go @@ -3,6 +3,7 @@ package main import ( "context" "flag" + "net/http" "os" "os/signal" "syscall" @@ -38,10 +39,17 @@ func main() { instanceID = flag.String("instance", uuid.New().String(), "the unique holder identity. used for leader lock") leaseLockName = flag.String("lease-lock-name", "k-rail-evicter-lock", "the lease lock resource name") leaseLockNamespace = flag.String("lease-lock-namespace", "k-rail", "the lease lock resource namespace") + probeServerAddress = flag.String("probe-listen-address", ":8080", "server address for healthz/readiness server") ) flag.Parse() flag.Set("logtostderr", "true") // glog: no disk log + defer func() { + if err := recover(); err != nil { + klog.Fatal(err) + } + }() + config, err := clientcmd.BuildConfigFromFlags(*master, *kubeconfig) if err != nil { klog.Fatal(err) @@ -56,6 +64,11 @@ func main() { defer cancel() watchSigTerm(cancel) + probeServer := newProbeServer(*probeServerAddress) + go func() { + klog.Fatal(probeServer.ListenAndServe()) + }() + br := record.NewBroadcaster() br.StartRecordingToSink(&corev1.EventSinkImpl{Interface: clientset.CoreV1().Events(metav1.NamespaceAll)}) br.StartLogging(glog.Infof) @@ -120,6 +133,9 @@ func main() { }, OnStoppedLeading: func() { // we can do cleanup here + cancel() + ctx, _ := context.WithTimeout(ctx, 100*time.Millisecond) + _ = probeServer.Shutdown(ctx) klog.Infof("Leader lost: %s", *instanceID) os.Exit(0) }, @@ -133,6 +149,15 @@ func main() { }) } +func newProbeServer(listenAddr string) http.Server { + okHandler := func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusOK) + } + http.HandleFunc("/healthz", okHandler) + http.HandleFunc("/readyness", okHandler) + return http.Server{Addr: listenAddr, Handler: nil} +} + func watchSigTerm(cancel context.CancelFunc) { ch := make(chan os.Signal, 1) signal.Notify(ch, os.Interrupt, syscall.SIGTERM)