Merge pull request #61 from cgreen12/master

Optionally test DNS resolution for each pod

pkg/goldpinger/client.go

@@ -16,6 +16,7 @@ package goldpinger
 
 import (
 	"fmt"
+	"net"
 	"sync"
 	"time"
 
@@ -27,7 +28,7 @@ import (
 
 // CheckNeighbours queries the kubernetes API server for all other goldpinger pods
 // then calls Ping() on each one
-func CheckNeighbours(ps *PodSelecter) models.CheckResults {
+func CheckNeighbours(ps *PodSelecter) *models.CheckResults {
 	return PingAllPods(ps.SelectPods())
 }
 
@@ -50,7 +51,25 @@ func pickPodHostIP(podIP, hostIP string) string {
 	return podIP
 }
 
-func PingAllPods(pods map[string]string) models.CheckResults {
+func checkDNS() *models.DNSResults {
+	results := models.DNSResults{}
+	for _, host := range GoldpingerConfig.DnsHosts{
+
+		var dnsResult models.DNSResult
+
+		start := time.Now()
+		_, err := net.LookupIP(host)
+		if err != nil {
+			dnsResult.Error = err.Error()
+			CountDnsError(host)
+		}
+		dnsResult.ResponseTimeMs = time.Since(start).Nanoseconds() / int64(time.Millisecond)
+		results[host] = dnsResult
+	}
+	return &results
+}
+
+func PingAllPods(pods map[string]string) *models.CheckResults {
 
 	result := models.CheckResults{}
 
@@ -84,11 +103,15 @@ func PingAllPods(pods map[string]string) models.CheckResults {
 			wg.Done()
 		}(podIP, hostIP)
 	}
+	if len(GoldpingerConfig.DnsHosts) > 0 {
+		result.DNSResults = *checkDNS()
+	}
 	wg.Wait()
 	close(ch)
 
 	counterHealthy, counterUnhealthy := 0.0, 0.0
 
+	result.PodResults = make(map[string]models.PodResult)
 	for response := range ch {
 		var podIPv4 strfmt.IPv4
 		podIPv4.UnmarshalText([]byte(response.podIP))
@@ -97,10 +120,10 @@ func PingAllPods(pods map[string]string) models.CheckResults {
 		} else {
 			counterUnhealthy++
 		}
-		result[response.podIP] = response.podResult
+		result.PodResults[response.podIP] = response.podResult
 	}
 	CountHealthyUnhealthyNodes(counterHealthy, counterUnhealthy)
-	return result
+	return &result
 }
 
 type CheckServicePodsResult struct {
@@ -161,6 +184,18 @@ func CheckAllPods(pods map[string]string) *models.CheckAllResults {
 			HostIP: response.hostIPv4,
 			PodIP:  podIPv4,
 		})
+		if response.checkAllPodResult.Response != nil &&
+		   response.checkAllPodResult.Response.DNSResults != nil {
+			if result.DNSResults == nil {
+				result.DNSResults = make(map[string]models.DNSResults)
+			}
+			for host := range response.checkAllPodResult.Response.DNSResults {
+				if result.DNSResults[host] == nil {
+					result.DNSResults[host] = make(map[string]models.DNSResult)
+				}
+				result.DNSResults[host][response.podIP] = response.checkAllPodResult.Response.DNSResults[host]
+			}
+		}
 	}
 	return &result
 }

pkg/goldpinger/config.go

@@ -31,4 +31,6 @@ var GoldpingerConfig = struct {
 	LabelSelector    string `long:"label-selector" description:"label selector to use to discover goldpinger pods in the cluster" env:"LABEL_SELECTOR" default:"app=goldpinger"`
 	KubernetesClient *kubernetes.Clientset
 	*PodSelecter
+
+	DnsHosts         []string `long:"host-to-resolve" description:"A host to attempt dns resolve on" env:"HOSTS_TO_RESOLVE" env-delim:" "`
 }{}

pkg/goldpinger/heatmap.go

@@ -115,7 +115,7 @@ func HeatmapHandler(w http.ResponseWriter, r *http.Request) {
 	// draw all the boxes
 	for sourceIP, results := range checkResults.Responses {
 		if *results.OK {
-			for destinationIP, response := range results.Response {
+			for destinationIP, response := range results.Response.PodResults {
 				x, y := getPingBoxCoordinates(order[sourceIP], order[destinationIP], boxSize, paddingSize)
 				color := getPingBoxColor(response.ResponseTimeMs, tresholdLatencies)
 				drawPingBox(canvas, boxSize+x, boxSize+y, boxSize, color)

pkg/goldpinger/stats.go

@@ -82,6 +82,16 @@ var (
 			"type",
 		},
 	)
+	goldpingerDnsErrorsCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "goldpinger_dns_errors_total",
+			Help: "Statistics of DNS errors per instance",
+		},
+		[]string{
+			"goldpinger_instance",
+			"host",
+		},
+	)
 
 	bootTime = time.Now()
 )
@@ -92,6 +102,7 @@ func init() {
 	prometheus.MustRegister(goldpingerResponseTimePeersHistogram)
 	prometheus.MustRegister(goldpingerResponseTimeKubernetesHistogram)
 	prometheus.MustRegister(goldpingerErrorsCounter)
+	prometheus.MustRegister(goldpingerDnsErrorsCounter)
 	log.Println("Metrics setup - see /metrics")
 }
 
@@ -131,6 +142,14 @@ func CountError(errorType string) {
 	).Inc()
 }
 
+// counts instances of dns errors
+func CountDnsError(host string) {
+	goldpingerDnsErrorsCounter.WithLabelValues(
+		GoldpingerConfig.Hostname,
+		host,
+	).Inc()
+}
+
 // returns a timer for easy observing of the durations of calls to kubernetes API
 func GetLabeledKubernetesCallsTimer() *prometheus.Timer {
 	return prometheus.NewTimer(

pkg/goldpinger/updater.go

@@ -31,7 +31,7 @@ func StartUpdater() {
 		for {
 			results := PingAllPods(GoldpingerConfig.PodSelecter.SelectPods())
 			var troublemakers []string
-			for podIP, value := range results {
+			for podIP, value := range results.PodResults {
 				if *value.OK != true {
 					troublemakers = append(troublemakers, fmt.Sprintf("%s (%s)", podIP, value.HostIP.String()))
 				}