From 1a3bbccd386e5a00fb1f8292eec52f95fb6c541d Mon Sep 17 00:00:00 2001 From: mojtaba-esk Date: Tue, 7 Jan 2025 16:14:06 +0330 Subject: [PATCH 1/2] feat: k8s auth token instead of kube conffig --- e2e/basic/suite_setup_test.go | 3 +- e2e/netshaper/suite_setup_test.go | 13 ++++++- e2e/sidecars/suite_setup_test.go | 9 ++++- e2e/suite.go | 17 +++++++++ e2e/system/suite_setup_test.go | 3 +- e2e/tshark/tshark_test.go | 10 +++-- pkg/k8s/clinet_options.go | 43 +++++++++++++++++++++ pkg/k8s/errors.go | 2 + pkg/k8s/k8s.go | 28 ++++---------- pkg/k8s/namespace.go | 4 +- pkg/k8s/pod.go | 14 +++---- pkg/k8s/utils.go | 24 +++++++++++- scripts/auth_token.sh | 62 +++++++++++++++++++++++++++++++ 13 files changed, 192 insertions(+), 40 deletions(-) create mode 100644 pkg/k8s/clinet_options.go create mode 100755 scripts/auth_token.sh diff --git a/e2e/basic/suite_setup_test.go b/e2e/basic/suite_setup_test.go index f52d74c..76e2fc9 100644 --- a/e2e/basic/suite_setup_test.go +++ b/e2e/basic/suite_setup_test.go @@ -35,7 +35,7 @@ func (s *Suite) SetupSuite() { logger = logrus.New() ) - k8sClient, err := k8s.NewClient(ctx, knuu.DefaultScope(), logger) + k8sClient, err := k8s.NewClient(ctx, knuu.DefaultScope(), logger, s.K8sDefaultOptions()...) s.Require().NoError(err) minioClient, err := minio.New(ctx, k8sClient, logger) @@ -46,6 +46,7 @@ func (s *Suite) SetupSuite() { K8sClient: k8sClient, MinioClient: minioClient, Timeout: testTimeout, + Logger: logger, }) s.Require().NoError(err) diff --git a/e2e/netshaper/suite_setup_test.go b/e2e/netshaper/suite_setup_test.go index 0beb515..fc3d9c4 100644 --- a/e2e/netshaper/suite_setup_test.go +++ b/e2e/netshaper/suite_setup_test.go @@ -4,9 +4,11 @@ import ( "context" "testing" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/suite" "github.com/celestiaorg/knuu/e2e" + "github.com/celestiaorg/knuu/pkg/k8s" "github.com/celestiaorg/knuu/pkg/knuu" ) @@ -19,11 +21,18 @@ func TestRunSuite(t *testing.T) { } func (s *Suite) SetupSuite() { - ctx := context.Background() + var ( + ctx = context.Background() + logger = logrus.New() + ) + + k8sClient, err := k8s.NewClient(ctx, knuu.DefaultScope(), logger, s.K8sDefaultOptions()...) + s.Require().NoError(err) - var err error s.Knuu, err = knuu.New(ctx, knuu.Options{ ProxyEnabled: true, + K8sClient: k8sClient, + Logger: logger, }) s.Require().NoError(err) s.T().Logf("Scope: %s", s.Knuu.Scope) diff --git a/e2e/sidecars/suite_setup_test.go b/e2e/sidecars/suite_setup_test.go index 3d52ee9..78efad7 100644 --- a/e2e/sidecars/suite_setup_test.go +++ b/e2e/sidecars/suite_setup_test.go @@ -10,6 +10,7 @@ import ( "github.com/celestiaorg/knuu/e2e" "github.com/celestiaorg/knuu/pkg/instance" + "github.com/celestiaorg/knuu/pkg/k8s" "github.com/celestiaorg/knuu/pkg/knuu" ) @@ -33,9 +34,13 @@ func (s *Suite) SetupSuite() { err error ) + k8sClient, err := k8s.NewClient(ctx, knuu.DefaultScope(), logger, s.K8sDefaultOptions()...) + s.Require().NoError(err) + s.Knuu, err = knuu.New(ctx, knuu.Options{ - Timeout: testTimeout, - Logger: logger, + K8sClient: k8sClient, + Timeout: testTimeout, + Logger: logger, }) s.Require().NoError(err) diff --git a/e2e/suite.go b/e2e/suite.go index ab89b53..c547599 100644 --- a/e2e/suite.go +++ b/e2e/suite.go @@ -3,6 +3,7 @@ package e2e import ( "context" "fmt" + "os" "sync" "sync/atomic" "time" @@ -11,6 +12,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" "github.com/celestiaorg/knuu/pkg/instance" + "github.com/celestiaorg/knuu/pkg/k8s" "github.com/celestiaorg/knuu/pkg/knuu" ) @@ -93,3 +95,18 @@ func (s *Suite) RetryOperation(operation func() error, maxRetries int) error { } return fmt.Errorf("operation failed after %d retries: %w", maxRetries, err) } + +func (s *Suite) K8sDefaultOptions() []k8s.Option { + if os.Getenv("K8S_AUTH_TOKEN") == "" { + s.T().Log("K8S_AUTH_TOKEN is not set, using default cluster config from ~/.kube/config") + return nil + } + + return []k8s.Option{ + k8s.WithAuthToken( + os.Getenv("K8S_HOST"), + os.Getenv("K8S_CA_CERT"), + os.Getenv("K8S_AUTH_TOKEN"), + ), + } +} diff --git a/e2e/system/suite_setup_test.go b/e2e/system/suite_setup_test.go index d851992..992440c 100644 --- a/e2e/system/suite_setup_test.go +++ b/e2e/system/suite_setup_test.go @@ -36,7 +36,7 @@ func (s *Suite) SetupSuite() { logger = logrus.New() ) - k8sClient, err := k8s.NewClient(ctx, knuu.DefaultScope(), logger) + k8sClient, err := k8s.NewClient(ctx, knuu.DefaultScope(), logger, s.K8sDefaultOptions()...) s.Require().NoError(err, "Error creating k8s client") minioClient, err := minio.New(ctx, k8sClient, logger) @@ -47,6 +47,7 @@ func (s *Suite) SetupSuite() { K8sClient: k8sClient, MinioClient: minioClient, // needed for build from git tests Timeout: testTimeout, + Logger: logger, }) s.Require().NoError(err) diff --git a/e2e/tshark/tshark_test.go b/e2e/tshark/tshark_test.go index 2acdebd..695f2cf 100644 --- a/e2e/tshark/tshark_test.go +++ b/e2e/tshark/tshark_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/api/resource" + "github.com/celestiaorg/knuu/e2e" "github.com/celestiaorg/knuu/pkg/k8s" "github.com/celestiaorg/knuu/pkg/knuu" "github.com/celestiaorg/knuu/pkg/minio" @@ -32,10 +33,13 @@ func TestTshark(t *testing.T) { t.Parallel() // Setup - ctx := context.Background() + var ( + ctx = context.Background() + logger = logrus.New() + s = e2e.Suite{} + ) - logger := logrus.New() - k8sClient, err := k8s.NewClient(ctx, knuu.DefaultScope(), logger) + k8sClient, err := k8s.NewClient(ctx, knuu.DefaultScope(), logger, s.K8sDefaultOptions()...) require.NoError(t, err, "error creating k8s client") minioClient, err := minio.New(ctx, k8sClient, logger) diff --git a/pkg/k8s/clinet_options.go b/pkg/k8s/clinet_options.go new file mode 100644 index 0000000..d1e6827 --- /dev/null +++ b/pkg/k8s/clinet_options.go @@ -0,0 +1,43 @@ +package k8s + +import "k8s.io/client-go/rest" + +type ClientOptions struct { + clusterDomain string + clusterHost string + authToken string + cert string + clusterConfig *rest.Config +} + +type Option func(*ClientOptions) + +func WithClusterDomain(clusterDomain string) Option { + return func(o *ClientOptions) { + o.clusterDomain = clusterDomain + } +} + +func WithAuthToken(host, cert, authToken string) Option { + return func(o *ClientOptions) { + o.clusterHost = host + o.authToken = authToken + o.cert = cert + } +} + +func WithClusterConfig(clusterConfig *rest.Config) Option { + return func(o *ClientOptions) { + o.clusterConfig = clusterConfig + } +} + +func getAppliedOptions(options ...Option) *ClientOptions { + opts := &ClientOptions{ + clusterDomain: defaultClusterDomain, + } + for _, opt := range options { + opt(opts) + } + return opts +} diff --git a/pkg/k8s/errors.go b/pkg/k8s/errors.go index d79833a..5bc8d4d 100644 --- a/pkg/k8s/errors.go +++ b/pkg/k8s/errors.go @@ -147,4 +147,6 @@ var ( ErrNoPortsFoundForService = errors.New("NoPortsFoundForService", "no ports found for service %s") ErrNoValidNodeIPFound = errors.New("NoValidNodeIPFound", "no valid node IP found for service %s") ErrInvalidClusterDomain = errors.New("InvalidClusterDomain", "invalid cluster domain `%s`") + ErrEmptyClusterHostOrAuthTokenOrCert = errors.New("EmptyClusterHostOrAuthTokenOrCert", "cluster host, auth token or cert is not set") + ErrClusterConfigNotSet = errors.New("ClusterConfigNotSet", "cluster config is not set") ) diff --git a/pkg/k8s/k8s.go b/pkg/k8s/k8s.go index ca944a7..5a34f74 100644 --- a/pkg/k8s/k8s.go +++ b/pkg/k8s/k8s.go @@ -9,6 +9,7 @@ import ( "k8s.io/client-go/discovery" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" ) const ( @@ -40,6 +41,7 @@ const ( type Client struct { clientset kubernetes.Interface discoveryClient discovery.DiscoveryInterface + clusterConfig *rest.Config dynamicClient dynamic.Interface namespace string clusterDomain string @@ -49,22 +51,10 @@ type Client struct { maxPendingDuration time.Duration } -type ClientOptions struct { - clusterDomain string -} - -type Option func(*ClientOptions) - -func WithClusterDomain(clusterDomain string) Option { - return func(o *ClientOptions) { - o.clusterDomain = clusterDomain - } -} - var _ KubeManager = &Client{} func NewClient(ctx context.Context, namespace string, logger *logrus.Logger, options ...Option) (*Client, error) { - config, err := getClusterConfig() + config, err := getClusterConfig(getAppliedOptions(options...)) if err != nil { return nil, ErrRetrievingKubernetesConfig.Wrap(err) } @@ -88,7 +78,8 @@ func NewClient(ctx context.Context, namespace string, logger *logrus.Logger, opt return nil, ErrCreatingDynamicClient.Wrap(err) } - return NewClientCustom(ctx, cs, dc, dC, namespace, logger, options...) + return NewClientCustom(ctx, cs, dc, dC, namespace, logger, + append(options, WithClusterConfig(config))...) } func NewClientCustom( @@ -100,13 +91,7 @@ func NewClientCustom( logger *logrus.Logger, options ...Option, ) (*Client, error) { - opts := &ClientOptions{ - clusterDomain: defaultClusterDomain, - } - for _, opt := range options { - opt(opts) - } - + opts := getAppliedOptions(options...) if err := validateDNS1123Subdomain( opts.clusterDomain, ErrInvalidClusterDomain.WithParams(opts.clusterDomain), @@ -122,6 +107,7 @@ func NewClientCustom( logger: logger, terminated: false, maxPendingDuration: defaultMaxPendingDuration, + clusterConfig: opts.clusterConfig, } kc.namespace = SanitizeName(namespace) if err := kc.CreateNamespace(ctx, kc.namespace); err != nil { diff --git a/pkg/k8s/namespace.go b/pkg/k8s/namespace.go index 308922e..2931853 100644 --- a/pkg/k8s/namespace.go +++ b/pkg/k8s/namespace.go @@ -2,6 +2,7 @@ package k8s import ( "context" + "fmt" corev1 "k8s.io/api/core/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" @@ -25,7 +26,8 @@ func (c *Client) CreateNamespace(ctx context.Context, name string) error { _, err := c.clientset.CoreV1().Namespaces().Create(ctx, namespace, metav1.CreateOptions{}) if err != nil { if !apierrs.IsAlreadyExists(err) { - return ErrCreatingNamespace.WithParams(name).Wrap(err) + fmt.Printf("err: %v\n", err) + return err } c.logger.WithField("name", name).Debug("namespace already exists, continuing") return nil diff --git a/pkg/k8s/pod.go b/pkg/k8s/pod.go index 035923a..75dd3d9 100644 --- a/pkg/k8s/pod.go +++ b/pkg/k8s/pod.go @@ -210,11 +210,10 @@ func (c *Client) RunCommandInPod( }, scheme.ParameterCodec) // Create an executor for the command execution - k8sConfig, err := getClusterConfig() - if err != nil { - return "", ErrGettingK8sConfig.Wrap(err) + if c.clusterConfig == nil { + return "", ErrClusterConfigNotSet } - exec, err := remotecommand.NewSPDYExecutor(k8sConfig, http.MethodPost, req.URL()) + exec, err := remotecommand.NewSPDYExecutor(c.clusterConfig, http.MethodPost, req.URL()) if err != nil { return "", ErrCreatingExecutor.Wrap(err) } @@ -283,9 +282,8 @@ func (c *Client) PortForwardPod( return ErrGettingPod.WithParams(podName).Wrap(err) } - restConfig, err := getClusterConfig() - if err != nil { - return ErrGettingClusterConfig.Wrap(err) + if c.clusterConfig == nil { + return ErrClusterConfigNotSet } url := c.clientset.CoreV1().RESTClient().Post(). @@ -295,7 +293,7 @@ func (c *Client) PortForwardPod( SubResource("portforward"). URL() - transport, upgrader, err := spdy.RoundTripperFor(restConfig) + transport, upgrader, err := spdy.RoundTripperFor(c.clusterConfig) if err != nil { return ErrCreatingRoundTripper.Wrap(err) } diff --git a/pkg/k8s/utils.go b/pkg/k8s/utils.go index 527481a..d4b71e9 100644 --- a/pkg/k8s/utils.go +++ b/pkg/k8s/utils.go @@ -24,13 +24,35 @@ func fileExists(path string) bool { } // getClusterConfig returns the appropriate Kubernetes cluster configuration. -func getClusterConfig() (*rest.Config, error) { +func getClusterConfig(opts *ClientOptions) (*rest.Config, error) { + if opts.clusterConfig != nil { + return opts.clusterConfig, nil + } + if isClusterEnvironment() { return rest.InClusterConfig() } + + if opts.authToken != "" { + return getClusterConfigWithToken(opts) + } return clientcmd.BuildConfigFromFlags("", kubeconfig) } +func getClusterConfigWithToken(opts *ClientOptions) (*rest.Config, error) { + if opts.clusterHost == "" || opts.authToken == "" || opts.cert == "" { + return nil, ErrEmptyClusterHostOrAuthTokenOrCert + } + + return &rest.Config{ + Host: opts.clusterHost, + BearerToken: opts.authToken, + TLSClientConfig: rest.TLSClientConfig{ + CAData: []byte(opts.cert), + }, + }, nil +} + // precompile the regular expression to avoid recompiling it on every function call var invalidCharsRegexp = regexp.MustCompile(`[^a-z0-9-]+`) diff --git a/scripts/auth_token.sh b/scripts/auth_token.sh new file mode 100755 index 0000000..936de2d --- /dev/null +++ b/scripts/auth_token.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# This script generates a bearer token for the knuu service account. +# It is used to authenticate the knuu service account to the Kubernetes API server. +# The token is used to test things manually. +# It requries ~/.kube/config to be set up. + +# Variables +SERVICE_ACCOUNT_NAME="knuu-service-account" +NAMESPACE="default" +ROLE_BINDING_NAME="knuu-rolebinding" + +cleanup() { + echo "Cleaning up resources..." + kubectl delete clusterrolebinding $ROLE_BINDING_NAME --ignore-not-found + kubectl delete serviceaccount $SERVICE_ACCOUNT_NAME --namespace=$NAMESPACE --ignore-not-found +} +trap cleanup EXIT + +# Create ServiceAccount +echo "Creating ServiceAccount..." +kubectl delete serviceaccount $SERVICE_ACCOUNT_NAME --namespace=$NAMESPACE --ignore-not-found +kubectl create serviceaccount $SERVICE_ACCOUNT_NAME --namespace=$NAMESPACE + +# Bind the cluster-admin role to the ServiceAccount +echo "Binding cluster-admin role..." +kubectl delete clusterrolebinding $ROLE_BINDING_NAME --ignore-not-found +kubectl create clusterrolebinding $ROLE_BINDING_NAME \ + --clusterrole=cluster-admin \ + --serviceaccount=$NAMESPACE:$SERVICE_ACCOUNT_NAME 2>/dev/null || \ +kubectl replace clusterrolebinding $ROLE_BINDING_NAME \ + --clusterrole=cluster-admin \ + --serviceaccount=$NAMESPACE:$SERVICE_ACCOUNT_NAME + +# Generate token +TOKEN=$(kubectl create token $SERVICE_ACCOUNT_NAME --namespace=$NAMESPACE) +if [ -z "$TOKEN" ]; then + echo "Failed to generate token!" + exit 1 +fi + +# Get API server URL +API_SERVER=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}') + +# Get CA Certificate +CA_CERT=$(kubectl config view --raw -o jsonpath='{.clusters[0].cluster.certificate-authority-data}' | base64 --decode) + +# Export variables +export K8S_HOST=$API_SERVER +export K8S_AUTH_TOKEN=$TOKEN +export K8S_CA_CERT="$CA_CERT" + +# Output +echo "=============================" +echo "Bearer Token: $TOKEN" +echo "API Server: $API_SERVER" +echo "CA Certificate: $CA_CERT" +echo "=============================" + +# Cleanup happens automatically on script exit. + +# Example how to run the tests using the auth token: +# . ./scripts/auth_token.sh && go test -v ./e2e/basic/ From b81bf312f3ec188da1bd151a767f6c5ea2704c51 Mon Sep 17 00:00:00 2001 From: mojtaba-esk Date: Tue, 7 Jan 2025 16:32:55 +0330 Subject: [PATCH 2/2] fix: a nit --- e2e/suite.go | 14 +++++++++----- pkg/k8s/namespace.go | 2 -- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/e2e/suite.go b/e2e/suite.go index c547599..74c1f0d 100644 --- a/e2e/suite.go +++ b/e2e/suite.go @@ -23,6 +23,10 @@ const ( nginxImage = "docker.io/nginx:latest" nginxVolumeOwner = 0 + + envK8sHost = "K8S_HOST" + envK8sCACert = "K8S_CA_CERT" + envK8sAuthToken = "K8S_AUTH_TOKEN" ) type Suite struct { @@ -97,16 +101,16 @@ func (s *Suite) RetryOperation(operation func() error, maxRetries int) error { } func (s *Suite) K8sDefaultOptions() []k8s.Option { - if os.Getenv("K8S_AUTH_TOKEN") == "" { - s.T().Log("K8S_AUTH_TOKEN is not set, using default cluster config from ~/.kube/config") + if os.Getenv(envK8sAuthToken) == "" || os.Getenv(envK8sCACert) == "" || os.Getenv(envK8sHost) == "" { + s.T().Logf("%s, %s and/or %s are not set, using default cluster config from ~/.kube/config", envK8sAuthToken, envK8sCACert, envK8sHost) return nil } return []k8s.Option{ k8s.WithAuthToken( - os.Getenv("K8S_HOST"), - os.Getenv("K8S_CA_CERT"), - os.Getenv("K8S_AUTH_TOKEN"), + os.Getenv(envK8sHost), + os.Getenv(envK8sCACert), + os.Getenv(envK8sAuthToken), ), } } diff --git a/pkg/k8s/namespace.go b/pkg/k8s/namespace.go index 2931853..c0f07ec 100644 --- a/pkg/k8s/namespace.go +++ b/pkg/k8s/namespace.go @@ -2,7 +2,6 @@ package k8s import ( "context" - "fmt" corev1 "k8s.io/api/core/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" @@ -26,7 +25,6 @@ func (c *Client) CreateNamespace(ctx context.Context, name string) error { _, err := c.clientset.CoreV1().Namespaces().Create(ctx, namespace, metav1.CreateOptions{}) if err != nil { if !apierrs.IsAlreadyExists(err) { - fmt.Printf("err: %v\n", err) return err } c.logger.WithField("name", name).Debug("namespace already exists, continuing")