Merge pull request #2299 from loft-sh/backport/v0.21/pr-2297

[v0.21] Merge pull request #2297 from neogopher/fix-set-etcd-headless-endpoints-correctly
loft-sh · Nov 28, 2024 · 585f1fb · 585f1fb
2 parents 65ee712 + c2f7f12
commit 585f1fb
Show file tree

Hide file tree

Showing 11 changed files with 33 additions and 18 deletions.
diff --git a/chart/templates/etcd-headless-service.yaml b/chart/templates/etcd-headless-service.yaml
@@ -1,6 +1,5 @@
 {{- if not .Values.experimental.isolatedControlPlane.headless }}
 {{- if or .Values.controlPlane.backingStore.etcd.deploy.enabled (include "vcluster.etcd.embedded.migrate" .) }}
-{{- if .Values.controlPlane.backingStore.etcd.deploy.headlessService.enabled }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -33,4 +32,3 @@ spec:
     release: "{{ .Release.Name }}"
 {{- end }}
 {{- end }}
-{{- end }}
diff --git a/chart/values.schema.json b/chart/values.schema.json
@@ -1079,10 +1079,6 @@
     },
     "EtcdDeployHeadlessService": {
       "properties": {
-        "enabled": {
-          "type": "boolean",
-          "description": "Enabled defines if the etcd headless service should be deployed"
-        },
         "annotations": {
           "additionalProperties": {
             "type": "string"

diff --git a/chart/values.yaml b/chart/values.yaml
@@ -413,7 +413,6 @@ controlPlane:
           annotations: {}
         # HeadlessService holds options for the external etcd headless service.
         headlessService:
-          enabled: true
           annotations: {}
 
   # Proxy defines options for the virtual cluster control plane proxy that is used to do authentication and intercept requests.

diff --git a/config/config.go b/config/config.go
@@ -1175,9 +1175,6 @@ type EtcdDeployService struct {
 }
 
 type EtcdDeployHeadlessService struct {
-	// Enabled defines if the etcd headless service should be deployed
-	Enabled bool `json:"enabled,omitempty"`
-
 	// Annotations are extra annotations for the external etcd headless service
 	Annotations map[string]string `json:"annotations,omitempty"`
 }

diff --git a/config/legacyconfig/migrate.go b/config/legacyconfig/migrate.go
@@ -154,7 +154,6 @@ func convertEtcd(oldConfig EtcdValues, newConfig *config.Config) error {
 	if oldConfig.Disabled {
 		newConfig.ControlPlane.BackingStore.Etcd.Deploy.StatefulSet.Enabled = false
 		newConfig.ControlPlane.BackingStore.Etcd.Deploy.Service.Enabled = false
-		newConfig.ControlPlane.BackingStore.Etcd.Deploy.HeadlessService.Enabled = false
 	}
 	if oldConfig.ImagePullPolicy != "" {
 		newConfig.ControlPlane.BackingStore.Etcd.Deploy.StatefulSet.ImagePullPolicy = oldConfig.ImagePullPolicy

diff --git a/config/values.yaml b/config/values.yaml
@@ -217,7 +217,6 @@ controlPlane:
           enabled: true
           annotations: {}
         headlessService:
-          enabled: true
           annotations: {}
 
   proxy:

diff --git a/pkg/etcd/client.go b/pkg/etcd/client.go
@@ -57,8 +57,10 @@ func NewFromConfig(ctx context.Context, vConfig *config.VirtualClusterConfig) (C
 
 		if vConfig.ControlPlane.BackingStore.Etcd.Embedded.Enabled {
 			etcdEndpoints = "https://127.0.0.1:2379"
-		} else {
+		} else if vConfig.ControlPlane.BackingStore.Etcd.Deploy.Service.Enabled {
 			etcdEndpoints = "https://" + vConfig.Name + "-etcd:2379"
+		} else {
+			etcdEndpoints = "https://" + vConfig.Name + "-etcd-headless:2379"
 		}
 	} else if vConfig.Distro() == vconfig.K8SDistro {
 		etcdEndpoints = constants.K8sKineEndpoint

diff --git a/pkg/k0s/k0s.go b/pkg/k0s/k0s.go
@@ -66,7 +66,11 @@ spec:
   storage:
     etcd:
       externalCluster:
+        {{- if .Values.controlPlane.backingStore.etcd.deploy.service.enabled }}
         endpoints: ["{{ .Release.Name }}-etcd:2379"]
+        {{- else }}
+        endpoints: ["{{ .Release.Name }}-etcd-headless:2379"]
+        {{- end }}
         caFile: /data/k0s/pki/etcd/ca.crt
         etcdPrefix: "/registry"
         clientCertFile: /data/k0s/pki/apiserver-etcd-client.crt
@@ -96,11 +100,18 @@ func StartK0S(ctx context.Context, cancel context.CancelFunc, vConfig *config.Vi
 
 	// wait until etcd is up and running
 	if vConfig.ControlPlane.BackingStore.Etcd.Deploy.Enabled {
+		var etcdEndpoint string
+		if vConfig.ControlPlane.BackingStore.Etcd.Deploy.Service.Enabled {
+			etcdEndpoint = "https://" + vConfig.Name + "-etcd:2379"
+		} else {
+			etcdEndpoint = "https://" + vConfig.Name + "-etcd-headless:2379"
+		}
+
 		err := etcd.WaitForEtcd(ctx, &etcd.Certificates{
 			CaCert:     "/data/k0s/pki/etcd/ca.crt",
 			ServerCert: "/data/k0s/pki/apiserver-etcd-client.crt",
 			ServerKey:  "/data/k0s/pki/apiserver-etcd-client.key",
-		}, "https://"+vConfig.Name+"-etcd:2379")
+		}, etcdEndpoint)
 		if err != nil {
 			return err
 		}

diff --git a/pkg/k3s/k3s.go b/pkg/k3s/k3s.go
@@ -51,17 +51,24 @@ func StartK3S(ctx context.Context, vConfig *config.VirtualClusterConfig, service
 			args = append(args, "--kube-apiserver-arg=endpoint-reconciler-type=none")
 		}
 		if vConfig.ControlPlane.BackingStore.Etcd.Deploy.Enabled {
+			var etcdEndpoint string
+			if vConfig.ControlPlane.BackingStore.Etcd.Deploy.Service.Enabled {
+				etcdEndpoint = "https://" + vConfig.Name + "-etcd:2379"
+			} else {
+				etcdEndpoint = "https://" + vConfig.Name + "-etcd-headless:2379"
+			}
+
 			// wait until etcd is up and running
 			err := etcd.WaitForEtcd(ctx, &etcd.Certificates{
 				CaCert:     "/data/pki/etcd/ca.crt",
 				ServerCert: "/data/pki/apiserver-etcd-client.crt",
 				ServerKey:  "/data/pki/apiserver-etcd-client.key",
-			}, "https://"+vConfig.Name+"-etcd:2379")
+			}, etcdEndpoint)
 			if err != nil {
 				return err
 			}
 
-			args = append(args, "--datastore-endpoint=https://"+vConfig.Name+"-etcd:2379")
+			args = append(args, "--datastore-endpoint="+etcdEndpoint)
 			args = append(args, "--datastore-cafile=/data/pki/etcd/ca.crt")
 			args = append(args, "--datastore-certfile=/data/pki/apiserver-etcd-client.crt")
 			args = append(args, "--datastore-keyfile=/data/pki/apiserver-etcd-client.key")

diff --git a/pkg/k8s/k8s.go b/pkg/k8s/k8s.go
@@ -86,8 +86,10 @@ func StartK8S(
 
 		if vConfig.ControlPlane.BackingStore.Etcd.Embedded.Enabled {
 			etcdEndpoints = "https://127.0.0.1:2379"
-		} else {
+		} else if vConfig.ControlPlane.BackingStore.Etcd.Deploy.Service.Enabled {
 			etcdEndpoints = "https://" + vConfig.Name + "-etcd:2379"
+		} else {
+			etcdEndpoints = "https://" + vConfig.Name + "-etcd-headless:2379"
 		}
 	}
 

diff --git a/pkg/setup/initialize.go b/pkg/setup/initialize.go
@@ -61,7 +61,11 @@ func initialize(ctx context.Context, parentCtx context.Context, options *config.
 	// migrate from
 	migrateFrom := ""
 	if options.ControlPlane.BackingStore.Etcd.Embedded.Enabled && options.ControlPlane.BackingStore.Etcd.Embedded.MigrateFromDeployedEtcd {
-		migrateFrom = "https://" + options.Name + "-etcd:2379"
+		if options.ControlPlane.BackingStore.Etcd.Deploy.Service.Enabled {
+			migrateFrom = "https://" + options.Name + "-etcd:2379"
+		} else {
+			migrateFrom = "https://" + options.Name + "-etcd-headless:2379"
+		}
 	}
 
 	// retrieve service cidr
@@ -228,6 +232,7 @@ func GenerateCerts(ctx context.Context, currentNamespaceClient kubernetes.Interf
 	etcdSans := []string{
 		"localhost",
 		etcdService,
+		etcdService + "-headless",
 		etcdService + "." + currentNamespace,
 		etcdService + "." + currentNamespace + ".svc",
 	}