Merge pull request #272 from weaveworks/rewrite-cert-rotate

rewrite cert rotation

Dockerfile

@@ -24,13 +24,13 @@ COPY utils/ utils/
 # Build
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o tf-controller cmd/manager/main.go
 
-FROM alpine:3.15.3
+FROM alpine:3.16
 
 LABEL org.opencontainers.image.source="https://github.com/weaveworks/tf-controller"
 
 RUN apk add --no-cache ca-certificates tini git openssh-client gnupg && \
-    apk add --no-cache libretls=3.3.4-r3 && \
-    apk add --no-cache busybox=1.34.1-r5
+    apk add --no-cache libretls && \
+    apk add --no-cache busybox
 
 COPY --from=builder /workspace/tf-controller /usr/local/bin/
 

api/v1alpha1/terraform_types.go

@@ -33,6 +33,7 @@ import (
 )
 
 const (
+	CACertSecretName = "tf-controller.tls"
 	// RunnerTLSSecretName is the name of the secret containing a TLS cert that will be written to
 	// the namespace in which a terraform runner is created
 	RunnerTLSSecretName   = "terraform-runner.tls"

cmd/manager/main.go

@@ -33,7 +33,6 @@ import (
 	infrav1 "github.com/weaveworks/tf-controller/api/v1alpha1"
 	"github.com/weaveworks/tf-controller/controllers"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
@@ -143,17 +142,15 @@ func main() {
 
 	certsReady := make(chan struct{})
 	rotator := &mtls.CertRotator{
-		CAName:         "tf-controller",
-		CAOrganization: "weaveworks",
-		DNSName:        "tf-controller",
-		SecretKey: types.NamespacedName{
-			Namespace: runtimeNamespace,
-			Name:      "tf-controller.tls",
-		},
-		Ready:                  certsReady,
-		CAValidityDuration:     caValidityDuration,
-		CertValidityDuration:   certValidityDuration,
-		RotationCheckFrequency: rotationCheckFrequency,
+		Ready:                         certsReady,
+		CAName:                        "tf-controller",
+		CAOrganization:                "weaveworks",
+		DNSName:                       "tf-controller",
+		CAValidityDuration:            caValidityDuration,
+		RotationCheckFrequency:        rotationCheckFrequency,
+		LookaheadInterval:             2 * time.Hour,
+		TriggerCARotation:             make(chan mtls.Trigger),
+		TriggerNamespaceTLSGeneration: make(chan mtls.Trigger),
 	}
 
 	const localHost = "localhost"
@@ -189,7 +186,13 @@ func main() {
 			Client: mgr.GetClient(),
 			Scheme: mgr.GetScheme(),
 		}
-		go mtls.StartGRPCServerForTesting(signalHandlerContext, runnerServer, "flux-system", "localhost:30000", mgr, rotator)
+		go func() {
+			err := mtls.StartGRPCServerForTesting(runnerServer, "flux-system", "localhost:30000", mgr, rotator)
+			if err != nil {
+				setupLog.Error(err, "unable to start runner server")
+				os.Exit(1)
+			}
+		}()
 	}
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {

cmd/runner/main.go

@@ -47,10 +47,12 @@ var (
 
 func main() {
 	var (
-		grpcPort int
+		grpcPort      int
+		tlsSecretName string
 	)
 
 	flag.IntVar(&grpcPort, "grpc-port", 30000, "The port on which to expose the grpc endpoint.")
+	flag.StringVar(&tlsSecretName, "tls-secret-name", "", "The TLS secret name.")
 	flag.Parse()
 
 	addr := fmt.Sprintf(":%d", grpcPort)
@@ -73,7 +75,7 @@ func main() {
 		signal.Stop(sigterm)
 	}()
 
-	err := mtls.RunnerServe(podNamespace, addr, sigterm)
+	err := mtls.RunnerServe(podNamespace, addr, tlsSecretName, sigterm)
 	if err != nil {
 		log.Fatal(err.Error())
 	}

controllers/suite_test.go

@@ -32,7 +32,6 @@ import (
 	"github.com/weaveworks/tf-controller/mtls"
 	"github.com/weaveworks/tf-controller/runner"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/types"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 
 	"github.com/onsi/gomega/ghttp"
@@ -91,8 +90,9 @@ func TestMain(m *testing.M) {
 	ctx, cancel = context.WithCancel(context.TODO())
 	// "bootstrapping test environment"
 	testEnv = &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "config", "crd", "bases")},
-		ErrorIfCRDPathMissing: true,
+		CRDDirectoryPaths:       []string{filepath.Join("..", "config", "crd", "bases")},
+		ErrorIfCRDPathMissing:   true,
+		ControlPlaneStopTimeout: 60 * time.Second,
 	}
 
 	cfg, err := testEnv.Start()
@@ -204,18 +204,15 @@ func TestMain(m *testing.M) {
 	certsReady := make(chan struct{})
 
 	rotator = &mtls.CertRotator{
-		CAName:         "localhost",
-		CAOrganization: "localhost",
-		DNSName:        "localhost",
-		SecretKey: types.NamespacedName{
-			Namespace: "flux-system",
-			Name:      "tf-controller.tls",
-		},
-		Ready:                  certsReady,
-		CAValidityDuration:     time.Hour * 24 * 7,
-		CertValidityDuration:   5*time.Minute + 1*time.Hour, // since the cert lookaheadInterval is set to 1 hour, using 1hr5m so that the cert is valid for 5 mins
-		RotationCheckFrequency: 10 * time.Second,
-		LookaheadInterval:      1 * time.Hour,
+		Ready:                         certsReady,
+		CAName:                        "localhost",
+		CAOrganization:                "localhost",
+		DNSName:                       "localhost",
+		CAValidityDuration:            time.Hour * 24 * 7,
+		RotationCheckFrequency:        10 * time.Second,
+		LookaheadInterval:             1 * time.Hour,
+		TriggerCARotation:             make(chan mtls.Trigger),
+		TriggerNamespaceTLSGeneration: make(chan mtls.Trigger),
 	}
 
 	if err := mtls.AddRotator(ctx, k8sManager, rotator); err != nil {
@@ -237,27 +234,36 @@ func TestMain(m *testing.M) {
 		panic(err.Error())
 	}
 
+	stopRunnerServer := make(chan os.Signal)
 	runnerServer = &runner.TerraformRunnerServer{
 		Client: k8sManager.GetClient(),
 		Scheme: k8sManager.GetScheme(),
+		Done:   stopRunnerServer,
 	}
 
-	go mtls.StartGRPCServerForTesting(ctx, runnerServer, "flux-system", "localhost:30000", k8sManager, rotator)
+	go func() {
+		err := mtls.StartGRPCServerForTesting(runnerServer, "flux-system", "localhost:30000", k8sManager, rotator)
+		if err != nil {
+			fmt.Println(err.Error())
+		}
+	}()
 
 	go func() {
 		if err := k8sManager.Start(ctx); err != nil {
-			panic(err.Error())
+			fmt.Println(err.Error())
 		}
 	}()
 
 	code := m.Run()
-	cancel()
+	// stopRunnerServer <- os.Interrupt
 	server.Close()
+	cancel()
+	close(stopRunnerServer)
 
 	// "tearing down the test environment"
 	err = testEnv.Stop()
 	if err != nil {
-		panic(err.Error())
+		fmt.Println(err.Error())
 	}
 
 	os.Exit(code)

controllers/tc009990_mtls_generate_creds_test.go

@@ -30,19 +30,14 @@ func Test_009990_mtls_generate_creds_test(t *testing.T) {
 	ctx := context.Background()
 	updatedTime := time.Now()
 
-	By("creating a secret and writing the certs")
-	caSecretKey := types.NamespacedName{
-		Namespace: "flux-system",
-		Name:      "tf-controller.tls",
-	}
-	caSecret := &corev1.Secret{}
-	g.Eventually(func() bool {
-		err := k8sClient.Get(ctx, caSecretKey, caSecret)
-		if err != nil {
-			return false
-		}
-		return len(caSecret.Data) == 4
-	}, timeout, interval).Should(BeTrue())
+	rotator.ResetCACache()
+	readyCh := make(chan *mtls.TriggerResult)
+	rotator.TriggerCARotation <- mtls.Trigger{Namespace: "", Ready: readyCh}
+	result := <-readyCh
+	g.Expect(result.Err).To(BeNil())
+
+	caSecret := result.Secret
+	g.Expect(len(caSecret.Data)).To(Equal(4))
 
 	It("should create the ca and runner certs")
 	g.Expect(caSecret.Data).Should(HaveKey("ca.crt"))
@@ -126,8 +121,12 @@ func Test_009990_mtls_generate_creds_test(t *testing.T) {
 	}, timeout, interval).Should(BeTrue())
 
 	By("checking that the runner secret gets created")
+
+	secretName, err := rotator.GetRunnerTLSSecretName()
+	g.Expect(err).Should(Succeed())
+
 	runnerSecret := &corev1.Secret{}
-	runnerSecretKey := types.NamespacedName{Namespace: "flux-system", Name: infrav1.RunnerTLSSecretName}
+	runnerSecretKey := types.NamespacedName{Namespace: "flux-system", Name: secretName}
 	g.Eventually(func() bool {
 		err := k8sClient.Get(ctx, runnerSecretKey, runnerSecret)
 		if err != nil {
@@ -139,7 +138,7 @@ func Test_009990_mtls_generate_creds_test(t *testing.T) {
 	By("verifying that the certificate authority cert is valid")
 	caCert := runnerSecret.Data["ca.crt"]
 	caKey := runnerSecret.Data["ca.key"]
-	caValid, err := mtls.ValidCert(caCert, caCert, caKey, rotator.CAName, time.Now())
+	caValid, err := mtls.ValidCert(caCert, caCert, caKey, rotator.CAName, nil, time.Now())
 	g.Expect(err).To(BeNil())
 	g.Expect(caValid).To(BeTrue())
 
@@ -152,35 +151,35 @@ func Test_009990_mtls_generate_creds_test(t *testing.T) {
 	tlsKey := runnerSecret.Data["tls.key"]
 	for _, ip := range []string{"172-1-0-1", "172-10-0-2", "172-127-0-3"} {
 		hostname := fmt.Sprintf("%s.flux-system.pod.cluster.local", ip)
-		tlsValid, err := mtls.ValidCert(caCert, tlsCert, tlsKey, hostname, time.Now())
+		tlsValid, err := mtls.ValidCert(caCert, tlsCert, tlsKey, hostname, nil, time.Now())
 		g.Expect(err).To(BeNil())
 		g.Expect(tlsValid).To(BeTrue())
 	}
 
 	By("verifying that the runner cert is valid only for the terraform runner namespace")
 	hostname := "172-1-0-1.kube-system.pod.cluster.local"
-	tlsValid, err := mtls.ValidCert(caCert, tlsCert, tlsKey, hostname, time.Now())
+	tlsValid, err := mtls.ValidCert(caCert, tlsCert, tlsKey, hostname, nil, time.Now())
 	g.Expect(err).ToNot(BeNil())
 	g.Expect(tlsValid).To(BeFalse())
 
 	By("rotating the CA should renew the server cert")
-	g.Expect(k8sClient.Delete(ctx, caSecret)).To(BeNil())
-
-	renewedCaSecret := &corev1.Secret{}
-	g.Eventually(func() bool {
-		err := k8sClient.Get(ctx, caSecretKey, renewedCaSecret)
-		if err != nil {
-			return false
-		}
-		return len(renewedCaSecret.Data) == 4
-	}, timeout, interval).Should(BeTrue())
+	rotator.ResetCACache()
+	renewedReadyCh := make(chan *mtls.TriggerResult)
+	rotator.TriggerCARotation <- mtls.Trigger{Namespace: "", Ready: renewedReadyCh}
+	renewedResult := <-renewedReadyCh
+	renewedCaSecret := renewedResult.Secret
 
 	g.Expect(bytes.Compare(caSecret.Data["ca.crt"], renewedCaSecret.Data["ca.crt"])).ToNot(BeZero())
 
 	By("checking that the runner secret gets updated")
+
+	renewedSecretName, err := rotator.GetRunnerTLSSecretName()
+	g.Expect(err).Should(Succeed())
+	renewedRunnerSecretKey := types.NamespacedName{Namespace: "flux-system", Name: renewedSecretName}
+
 	updatedRunnerSecret := &corev1.Secret{}
 	g.Eventually(func() int {
-		err := k8sClient.Get(ctx, runnerSecretKey, updatedRunnerSecret)
+		err := k8sClient.Get(ctx, renewedRunnerSecretKey, updatedRunnerSecret)
 		if err != nil {
 			return 1
 		}
@@ -194,7 +193,7 @@ func Test_009990_mtls_generate_creds_test(t *testing.T) {
 	caCert = renewedCaSecret.Data["ca.crt"]
 	tlsCert = updatedRunnerSecret.Data["tls.crt"]
 	tlsKey = updatedRunnerSecret.Data["tls.key"]
-	tlsValid, err = mtls.ValidCert(caCert, tlsCert, tlsKey, hostname, time.Now())
+	tlsValid, err = mtls.ValidCert(caCert, tlsCert, tlsKey, hostname, nil, time.Now())
 	g.Expect(err).To(BeNil())
-	g.Expect(tlsValid).To((BeTrue()))
+	g.Expect(tlsValid).To(BeTrue())
 }