Merge pull request #490 from FabianKramm/main

feat: improve kuberentes provider & k8s e2e tests

.github/workflows/e2e-tests.yaml

@@ -11,6 +11,10 @@ on:
       - "e2e/**_test.go" # include test files in e2e again
       - ".github/workflows/e2e-tests.yaml"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   GO111MODULE: on
   GOFLAGS: -mod=vendor
@@ -28,14 +32,27 @@ jobs:
       with:
         go-version: 1.19
 
+    - name: Set up kind k8s cluster
+      uses: engineerd/[email protected]
+      with:
+        version: "v0.20.0"
+        image: kindest/node:v1.27.3
+
+    - name: Testing kind cluster set-up
+      run: |
+        set -x
+        kubectl cluster-info
+        kubectl get pods -n kube-system -v 10
+        echo "kubectl config current-context:" $(kubectl config current-context)
+        echo "KUBECONFIG env var:" ${KUBECONFIG}
+
     - name: Build binary and copy to the E2E directory
       working-directory: ./e2e
       run: |
         chmod +x ../hack/build-e2e.sh
         BUILDDIR=bin SRCDIR=".." ../hack/build-e2e.sh
 
-
     - name: E2E test
       working-directory: ./e2e
       run: |
-        sudo go test -v -ginkgo.v -timeout 3600s
+        sudo KUBECONFIG=/home/runner/.kube/config go test -v -ginkgo.v -timeout 3600s

cmd/up.go

@@ -181,7 +181,7 @@ func (cmd *UpCmd) Run(ctx context.Context, devPodConfig *config.Config, client c
 
 func startVSCodeLocally(client client2.BaseWorkspaceClient, workspaceFolder string, ideOptions map[string]config.OptionValue, log log.Logger) error {
 	openURL := `vscode://vscode-remote/ssh-remote+` + client.Workspace() + `.devpod/` + url.QueryEscape(workspaceFolder)
-	if vscode.Options.GetValue(ideOptions, vscode.OpenSameWindow) == "false" {
+	if vscode.Options.GetValue(ideOptions, vscode.OpenNewWindow) == "true" {
 		openURL += "?windowId=_blank"
 	}
 

e2e/framework/command.go

@@ -186,6 +186,11 @@ func (f *Framework) DevPodMachineDelete(args []string) error {
 	}
 	return nil
 }
+func (f *Framework) DevPodWorkspaceStop(ctx context.Context, extraArgs ...string) error {
+	baseArgs := []string{"stop"}
+	baseArgs = append(baseArgs, extraArgs...)
+	return f.ExecCommandStdout(ctx, baseArgs)
+}
 
 func (f *Framework) DevPodWorkspaceDelete(ctx context.Context, workspace string, extraArgs ...string) error {
 	baseArgs := []string{"delete", workspace}

e2e/framework/exec.go

@@ -25,6 +25,18 @@ func (f *Framework) ExecCommandOutput(ctx context.Context, args []string) (strin
 	return execOut.String(), nil
 }
 
+// ExecCommandStdout executes the command string with the devpod test binary
+func (f *Framework) ExecCommandStdout(ctx context.Context, args []string) error {
+	cmd := exec.CommandContext(ctx, f.DevpodBinDir+"/"+f.DevpodBinName, args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // ExecCommand executes the command string with the devpod test binary
 func (f *Framework) ExecCommand(ctx context.Context, captureStdOut, searchForString bool, searchString string, args []string) error {
 	var execOut bytes.Buffer

e2e/tests/up/testdata/kubernetes/.devcontainer.json

@@ -0,0 +1,4 @@
+{
+	"name": "Go",
+	"image": "mcr.microsoft.com/devcontainers/go:0-1.19-bullseye"
+}

e2e/tests/up/up.go

@@ -2,6 +2,7 @@ package up
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -16,6 +17,7 @@ import (
 	"github.com/loft-sh/devpod/pkg/devcontainer/config"
 	docker "github.com/loft-sh/devpod/pkg/docker"
 	"github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
 
 	"github.com/loft-sh/devpod/e2e/framework"
 	"github.com/onsi/ginkgo/v2"
@@ -36,6 +38,80 @@ var _ = DevPodDescribe("devpod up test suite", func() {
 		framework.ExpectNoError(err)
 	})
 
+	ginkgo.It("run devpod in Kubernetes", func() {
+		ctx := context.Background()
+		f := framework.NewDefaultFramework(initialDir + "/bin")
+		tempDir, err := framework.CopyToTempDir("tests/up/testdata/kubernetes")
+		framework.ExpectNoError(err)
+		ginkgo.DeferCleanup(framework.CleanupTempDir, initialDir, tempDir)
+
+		err = f.DevPodProviderAdd(ctx, "kubernetes", "-o", "KUBERNETES_NAMESPACE=devpod")
+		framework.ExpectNoError(err)
+		ginkgo.DeferCleanup(func() {
+			err = f.DevPodProviderDelete(context.Background(), "kubernetes")
+			framework.ExpectNoError(err)
+		})
+
+		// run up
+		err = f.DevPodUp(ctx, tempDir)
+		framework.ExpectNoError(err)
+
+		// check pod is there
+		cmd := exec.Command("kubectl", "get", "pods", "-l", "devpod.sh/created=true", "-o", "json", "-n", "devpod")
+		stdout, err := cmd.Output()
+		framework.ExpectNoError(err)
+
+		// check if pod is there
+		list := &corev1.PodList{}
+		err = json.Unmarshal(stdout, list)
+		framework.ExpectNoError(err)
+		framework.ExpectEqual(len(list.Items), 1, "Expect 1 pod")
+		framework.ExpectEqual(len(list.Items[0].Spec.Containers), 1, "Expect 1 container")
+		framework.ExpectEqual(list.Items[0].Spec.Containers[0].Image, "mcr.microsoft.com/devcontainers/go:0-1.19-bullseye", "Expect container image")
+
+		// check if ssh works
+		err = f.DevPodSSHEchoTestString(ctx, tempDir)
+		framework.ExpectNoError(err)
+
+		// stop workspace
+		err = f.DevPodWorkspaceStop(ctx, tempDir)
+		framework.ExpectNoError(err)
+
+		// check pod is there
+		cmd = exec.Command("kubectl", "get", "pods", "-l", "devpod.sh/created=true", "-o", "json", "-n", "devpod")
+		stdout, err = cmd.Output()
+		framework.ExpectNoError(err)
+
+		// check if pod is there
+		list = &corev1.PodList{}
+		err = json.Unmarshal(stdout, list)
+		framework.ExpectNoError(err)
+		framework.ExpectEqual(len(list.Items), 0, "Expect no pods")
+
+		// run up
+		err = f.DevPodUp(ctx, tempDir)
+		framework.ExpectNoError(err)
+
+		// check pod is there
+		cmd = exec.Command("kubectl", "get", "pods", "-l", "devpod.sh/created=true", "-o", "json", "-n", "devpod")
+		stdout, err = cmd.Output()
+		framework.ExpectNoError(err)
+
+		// check if pod is there
+		list = &corev1.PodList{}
+		err = json.Unmarshal(stdout, list)
+		framework.ExpectNoError(err)
+		framework.ExpectEqual(len(list.Items), 1, "Expect 1 pod")
+
+		// check if ssh works
+		err = f.DevPodSSHEchoTestString(ctx, tempDir)
+		framework.ExpectNoError(err)
+
+		// delete workspace
+		err = f.DevPodWorkspaceDelete(ctx, tempDir)
+		framework.ExpectNoError(err)
+	})
+
 	ginkgo.Context("print error message correctly", func() {
 		ginkgo.It("make sure devpod output is correct and log-output works correctly", func(ctx context.Context) {
 			f := framework.NewDefaultFramework(initialDir + "/bin")

pkg/driver/kubernetes/build.go

@@ -156,9 +156,17 @@ func (k *kubernetesDriver) buildPod(
 	// get pod
 	var pod *corev1.Pod
 	if k.config.BuildkitPrivileged == "true" {
-		pod = getPrivilegedBuildKitPod(id, k.config.BuildkitImage)
+		pod = getPrivilegedBuildKitPod(id, k.config.BuildkitImage, parseResources(k.config.BuildkitResources, k.Log))
 	} else {
-		pod = getRootlessBuildKitPod(id, k.config.BuildkitImage)
+		pod = getRootlessBuildKitPod(id, k.config.BuildkitImage, parseResources(k.config.BuildkitResources, k.Log))
+	}
+
+	// parse node selector
+	if k.config.BuildkitNodeSelector != "" {
+		pod.Spec.NodeSelector, err = parseLabels(k.config.BuildkitNodeSelector)
+		if err != nil {
+			return nil, fmt.Errorf("parse node selector: %w", err)
+		}
 	}
 
 	// delete existing pod
@@ -259,7 +267,7 @@ func newBuildKitClient(ctx context.Context, reader io.Reader, writer io.WriteClo
 	}))
 }
 
-func getPrivilegedBuildKitPod(id, buildKitImage string) *corev1.Pod {
+func getPrivilegedBuildKitPod(id, buildKitImage string, resources corev1.ResourceRequirements) *corev1.Pod {
 	if buildKitImage == "" {
 		buildKitImage = defaultBuildkitImage
 	}
@@ -270,14 +278,16 @@ func getPrivilegedBuildKitPod(id, buildKitImage string) *corev1.Pod {
 			APIVersion: corev1.SchemeGroupVersion.String(),
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name: id + "-" + "buildkit",
+			Name:   id + "-" + "buildkit",
+			Labels: DevPodLabels,
 		},
 		Spec: corev1.PodSpec{
 			EnableServiceLinks: new(bool),
 			Containers: []corev1.Container{
 				{
-					Name:  "buildkitd",
-					Image: buildKitImage,
+					Name:      "buildkitd",
+					Image:     buildKitImage,
+					Resources: resources,
 					LivenessProbe: &corev1.Probe{
 						Handler: corev1.Handler{
 							Exec: &corev1.ExecAction{
@@ -313,7 +323,7 @@ func getPrivilegedBuildKitPod(id, buildKitImage string) *corev1.Pod {
 	}
 }
 
-func getRootlessBuildKitPod(id, buildKitImage string) *corev1.Pod {
+func getRootlessBuildKitPod(id, buildKitImage string, resources corev1.ResourceRequirements) *corev1.Pod {
 	if buildKitImage == "" {
 		buildKitImage = defaultRootlessBuildkitImage
 	}
@@ -324,7 +334,8 @@ func getRootlessBuildKitPod(id, buildKitImage string) *corev1.Pod {
 			APIVersion: corev1.SchemeGroupVersion.String(),
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name: id + "-" + "buildkit",
+			Name:   id + "-" + "buildkit",
+			Labels: DevPodLabels,
 			Annotations: map[string]string{
 				"container.apparmor.security.beta.kubernetes.io/buildkitd": "unconfined",
 			},
@@ -338,6 +349,7 @@ func getRootlessBuildKitPod(id, buildKitImage string) *corev1.Pod {
 					Args: []string{
 						"--oci-worker-no-process-sandbox",
 					},
+					Resources: resources,
 					LivenessProbe: &corev1.Probe{
 						Handler: corev1.Handler{
 							Exec: &corev1.ExecAction{

pkg/driver/kubernetes/helper.go

@@ -0,0 +1,90 @@
+package kubernetes
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/loft-sh/log"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+const (
+	limitsPrefix = "limits."
+)
+
+func parseResources(resourceString string, log log.Logger) corev1.ResourceRequirements {
+	if resourceString == "" {
+		return corev1.ResourceRequirements{}
+	}
+
+	resourcesSplitted := strings.Split(resourceString, ",")
+	requests := corev1.ResourceList{}
+	limits := corev1.ResourceList{}
+	for _, resourceName := range resourcesSplitted {
+		resourceName = strings.TrimSpace(resourceName)
+
+		// requests
+		if strings.HasPrefix(corev1.DefaultResourceRequestsPrefix, resourceName) {
+			strippedResource := strings.TrimPrefix(corev1.DefaultResourceRequestsPrefix, resourceName)
+			name, quantity, err := parseResource(strippedResource)
+			if err != nil {
+				log.Error(err.Error())
+				continue
+			}
+
+			requests[corev1.ResourceName(name)] = quantity
+		}
+
+		// limits
+		if strings.HasPrefix(limitsPrefix, resourceName) {
+			strippedResource := strings.TrimPrefix(limitsPrefix, resourceName)
+			name, quantity, err := parseResource(strippedResource)
+			if err != nil {
+				log.Error(err.Error())
+				continue
+			}
+
+			limits[corev1.ResourceName(name)] = quantity
+		}
+	}
+
+	return corev1.ResourceRequirements{
+		Limits:   limits,
+		Requests: requests,
+	}
+}
+
+func parseLabels(str string) (map[string]string, error) {
+	if str == "" {
+		return nil, nil
+	}
+
+	labels := strings.Split(str, ",")
+	retMap := map[string]string{}
+	for _, label := range labels {
+		label = strings.TrimSpace(label)
+		splitted := strings.SplitN(label, "=", 2)
+		if len(splitted) != 2 {
+			return nil, fmt.Errorf("invalid label '%s', expected format label=value", label)
+		}
+
+		retMap[splitted[0]] = splitted[1]
+	}
+
+	return retMap, nil
+}
+
+func parseResource(resourceName string) (string, resource.Quantity, error) {
+	splittedResource := strings.SplitN(resourceName, "=", 2)
+	if len(splittedResource) != 2 {
+		return "", resource.Quantity{}, fmt.Errorf("error parsing resource %s: expected form resource=quantity", resourceName)
+	}
+
+	quantity, err := resource.ParseQuantity(splittedResource[1])
+	if err != nil {
+		return "", resource.Quantity{}, fmt.Errorf("error parsing resource %s: %w", resourceName, err)
+	}
+
+	return splittedResource[0], quantity, nil
+}

pkg/driver/kubernetes/init.go → pkg/driver/kubernetes/init_container.go

@@ -40,6 +40,7 @@ func (k *kubernetesDriver) getInitContainer(
 			Image:        imageName,
 			Command:      []string{"/bin/sh"},
 			Args:         []string{"-c", strings.Join(commands, "\n") + "\n"},
+			Resources:    parseResources(k.config.HelperResources, k.Log),
 			VolumeMounts: volumeMounts,
 			SecurityContext: &corev1.SecurityContext{
 				RunAsUser:    &[]int64{0}[0],

pkg/driver/kubernetes/kubernetes.go

@@ -188,14 +188,6 @@ func (k *kubernetesDriver) DeleteDevContainer(ctx context.Context, id string, de
 		if err != nil {
 			return perrors.Wrapf(err, "delete role binding: %s", string(out))
 		}
-
-		if k.config.ServiceAccount == "" {
-			k.Log.Infof("Delete service account '%s'...", id)
-			out, err = k.buildCmd(ctx, []string{"delete", "serviceaccount", id, "--ignore-not-found"}).CombinedOutput()
-			if err != nil {
-				return perrors.Wrapf(err, "delete service account: %s", string(out))
-			}
-		}
 	}
 
 	return nil

pkg/driver/kubernetes/pvc.go

@@ -73,10 +73,8 @@ func (k *kubernetesDriver) buildPersistentVolumeClaim(
 			APIVersion: corev1.SchemeGroupVersion.String(),
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name: name,
-			Labels: map[string]string{
-				"devpod": "true",
-			},
+			Name:   name,
+			Labels: DevPodLabels,
 			Annotations: map[string]string{
 				DevContainerInfoAnnotation: containerInfo,
 			},