Close idle connections periodically to balance load through SLB

Collector connections (and other clients) use HTTP keep-alive which
prevents connections from being closed on the server.  This can cause
problems when scaling out ingestor pods because the existing connections
persist and are not balanced to new pods.

This decreases the idle timeout to forces connections to close so a new connection
can be established which can land on a different host.

cmd/ingestor/main.go

@@ -297,7 +297,12 @@ func realMain(ctx *cli.Context) error {
 	metricsMux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
 	metricsMux.HandleFunc("/debug/pprof/trace", pprof.Trace)
 
-	srv := &http.Server{Handler: mux}
+	srv := &http.Server{
+		Handler: mux,
+		// Close idle connections fairly often to establish new connections through the load balancer
+		// so that long-lived connections don't stay pinned to the same node indefinitely.
+		IdleTimeout: 15 * time.Second,
+	}
 	srv.ErrorLog = newLogger()
 
 	go func() {

collector/service.go

@@ -191,6 +191,8 @@ func (s *Service) scrape() {
 	s.wg.Add(1)
 	defer s.wg.Done()
 
+	reconnectTimer := time.NewTicker(5 * time.Minute)
+	defer reconnectTimer.Stop()
 	t := time.NewTicker(s.opts.ScrapeInterval)
 	defer t.Stop()
 	for {
@@ -199,6 +201,8 @@ func (s *Service) scrape() {
 			return
 		case <-t.C:
 			s.scrapeTargets()
+		case <-reconnectTimer.C:
+			s.remoteClient.CloseIdleConnections()
 		}
 	}
 }

pkg/promremote/client.go

@@ -21,7 +21,7 @@ func NewClient(timeout time.Duration, insecureSkipVerify bool) (*Client, error)
 	t := http.DefaultTransport.(*http.Transport).Clone()
 	t.MaxIdleConns = 100
 	t.MaxConnsPerHost = 100
-	t.MaxIdleConnsPerHost = 100
+	t.MaxIdleConnsPerHost = 5
 	t.ResponseHeaderTimeout = timeout
 	t.IdleConnTimeout = time.Minute
 	t.TLSClientConfig.InsecureSkipVerify = insecureSkipVerify
@@ -55,8 +55,6 @@ func (c *Client) Write(ctx context.Context, endpoint string, wr *prompb.WriteReq
 	req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
 	req.Header.Set("User-Agent", "adx-mon")
 
-	// req.Close = true
-
 	resp, err := c.httpClient.Do(req)
 	if err != nil {
 		return fmt.Errorf("http post: %w", err)
@@ -75,3 +73,7 @@ func (c *Client) Write(ctx context.Context, endpoint string, wr *prompb.WriteReq
 	}
 	return nil
 }
+
+func (c *Client) CloseIdleConnections() {
+	c.httpClient.CloseIdleConnections()
+}