diff --git a/pkg/ext/apiserver.go b/pkg/ext/apiserver.go index 3b7cd87f0..ea0dc452d 100644 --- a/pkg/ext/apiserver.go +++ b/pkg/ext/apiserver.go @@ -7,6 +7,7 @@ import ( "net/http" "strings" "sync" + "time" "k8s.io/apimachinery/pkg/api/meta" metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" @@ -21,6 +22,7 @@ import ( "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/dynamiccertificates" genericoptions "k8s.io/apiserver/pkg/server/options" utilversion "k8s.io/apiserver/pkg/util/version" openapicommon "k8s.io/kube-openapi/pkg/common" @@ -44,6 +46,15 @@ type ExtensionAPIServerOptions struct { // Authenticator will be used to authenticate requests coming to the // extension API server. Required. + // + // If the authenticator implements [dynamiccertificates.CAContentProvider], the + // ClientCA will be set on the underlying SecureServing struct. If the authenticator + // implements [dynamiccertificates.ControllerRunner] too, then Run() will be called so + // that the authenticators can run in the background. (See DefaultAuthenticator for + // example). + // + // Use a UnionAuthenticator to have multiple ways of authenticating requests. See + // [NewUnionAuthenticator] for an example. Authenticator authenticator.Request // Authorizer will be used to authorize requests based on the user, @@ -160,6 +171,9 @@ func NewExtensionAPIServer(scheme *runtime.Scheme, codecs serializer.CodecFactor } config.Authentication.Authenticator = opts.Authenticator + if caContentProvider, ok := opts.Authenticator.(dynamiccertificates.CAContentProvider); ok { + config.SecureServing.ClientCA = caContentProvider + } completedConfig := config.Complete() genericServer, err := completedConfig.New("imperative-api", genericapiserver.NewEmptyDelegate()) @@ -188,6 +202,11 @@ func (s *ExtensionAPIServer) Run(ctx context.Context) error { } } prepared := s.genericAPIServer.PrepareRun() + + if _, _, err := prepared.NonBlockingRunWithContext(ctx, time.Second*5); err != nil { + return err + } + s.handlerMu.Lock() s.handler = prepared.Handler s.handlerMu.Unlock() diff --git a/pkg/ext/apiserver_authentication.go b/pkg/ext/apiserver_authentication.go new file mode 100644 index 000000000..906c0b7b6 --- /dev/null +++ b/pkg/ext/apiserver_authentication.go @@ -0,0 +1,257 @@ +package ext + +import ( + "context" + "crypto/x509" + "fmt" + "net/http" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/apis/apiserver" + "k8s.io/apiserver/pkg/authentication/authenticator" + "k8s.io/apiserver/pkg/authentication/authenticatorfactory" + "k8s.io/apiserver/pkg/authentication/request/headerrequest" + authenticatorunion "k8s.io/apiserver/pkg/authentication/request/union" + "k8s.io/apiserver/pkg/server/dynamiccertificates" + "k8s.io/apiserver/pkg/server/options" + "k8s.io/client-go/kubernetes" +) + +var _ dynamiccertificates.ControllerRunner = &UnionAuthenticator{} +var _ dynamiccertificates.CAContentProvider = &UnionAuthenticator{} + +// UnionAuthenticator chains authenticators together to allow many ways of authenticating +// requests for the extension API server. For example, we might want to use Rancher's +// token authentication and fallback to the default authentication (mTLS) defined +// by Kubernetes. +// +// UnionAuthenticator is both a [dynamiccertificates.ControllerRunner] and a +// [dynamiccertificates.CAContentProvider]. +type UnionAuthenticator struct { + unionAuthenticator authenticator.Request + unionCAContentProvider dynamiccertificates.CAContentProvider +} + +// NewUnionAuthenticator creates a [UnionAuthenticator]. +// +// The authenticators will be tried one by one, in the order they are given, until +// one succeed or all fails. +// +// Here's an example usage: +// +// customAuth := authenticator.RequestFunc(func(req *http.Request) (*Response, bool, error) { +// // use request to determine what the user is, otherwise return false +// }) +// default, err := NewDefaultAuthenticator(client) +// if err != nil { +// return err +// } +// auth := NewUnionAuthenticator(customAuth, default) +// err = auth.RunOnce(ctx) +func NewUnionAuthenticator(authenticators ...authenticator.Request) *UnionAuthenticator { + caContentProviders := make([]dynamiccertificates.CAContentProvider, 0, len(authenticators)) + for _, auth := range authenticators { + auth, ok := auth.(dynamiccertificates.CAContentProvider) + if !ok { + continue + } + caContentProviders = append(caContentProviders, auth) + } + return &UnionAuthenticator{ + unionAuthenticator: authenticatorunion.New(authenticators...), + unionCAContentProvider: dynamiccertificates.NewUnionCAContentProvider(caContentProviders...), + } +} + +// AuthenticateRequest implements [authenticator.Request] +func (u *UnionAuthenticator) AuthenticateRequest(req *http.Request) (*authenticator.Response, bool, error) { + return u.unionAuthenticator.AuthenticateRequest(req) +} + +// AuthenticateRequest implements [dynamiccertificates.Notifier] +// This is part of the [dynamiccertificates.CAContentProvider] interface. +func (u *UnionAuthenticator) AddListener(listener dynamiccertificates.Listener) { + u.unionCAContentProvider.AddListener(listener) +} + +// AuthenticateRequest implements [dynamiccertificates.CAContentProvider] +func (u *UnionAuthenticator) Name() string { + return u.unionCAContentProvider.Name() +} + +// AuthenticateRequest implements [dynamiccertificates.CAContentProvider] +func (u *UnionAuthenticator) CurrentCABundleContent() []byte { + return u.unionCAContentProvider.CurrentCABundleContent() +} + +// AuthenticateRequest implements [dynamiccertificates.CAContentProvider] +func (u *UnionAuthenticator) VerifyOptions() (x509.VerifyOptions, bool) { + return u.unionCAContentProvider.VerifyOptions() +} + +// AuthenticateRequest implements [dynamiccertificates.CAContentProvider] +func (u *UnionAuthenticator) RunOnce(ctx context.Context) error { + runner, ok := u.unionCAContentProvider.(dynamiccertificates.ControllerRunner) + if !ok { + return nil + } + return runner.RunOnce(ctx) +} + +// AuthenticateRequest implements [dynamiccertificates.CAContentProvider] +func (u *UnionAuthenticator) Run(ctx context.Context, workers int) { + runner, ok := u.unionCAContentProvider.(dynamiccertificates.ControllerRunner) + if !ok { + return + } + + runner.Run(ctx, workers) +} + +const ( + authenticationConfigMapNamespace = metav1.NamespaceSystem + authenticationConfigMapName = "extension-apiserver-authentication" +) + +var _ dynamiccertificates.ControllerRunner = &DefaultAuthenticator{} +var _ dynamiccertificates.CAContentProvider = &DefaultAuthenticator{} + +// DefaultAuthenticator is an [authenticator.Request] that authenticates a user by: +// - making sure the client uses a certificate signed by the CA defined in the +// `extension-apiserver-authentication` configmap in the `kube-system` namespace and +// - making sure the CN of the cert is part of the allow list, also defined in the same configmap +// +// This authentication is better explained in https://kubernetes.io/docs/tasks/extend-kubernetes/configure-aggregation-layer/ +// +// This authenticator is a [dynamiccertificates.ControllerRunner] which means +// it will run in the background to dynamically watch the content of the configmap. +// +// When using the DefaultAuthenticator, it is suggested to call RunOnce() to initialize +// the CA state. It is also possible to watch for changes to the CA bundle with the AddListener() +// method. Here's an example usage: +// +// auth, err := NewDefaultAuthenticator(client) +// if err != nil { +// return err +// } +// auth.AddListener(myListener{auth: auth}) // myListener should react to CA bundle changes +// err = auth.RunOnce(ctx) +type DefaultAuthenticator struct { + requestHeaderConfig *authenticatorfactory.RequestHeaderConfig + authenticator authenticator.Request +} + +// NewDefaultAuthenticator creates a DefaultAuthenticator +func NewDefaultAuthenticator(client kubernetes.Interface) (*DefaultAuthenticator, error) { + requestHeaderConfig, err := createRequestHeaderConfig(client) + if err != nil { + return nil, fmt.Errorf("requestheaderconfig: %w", err) + } + + cfg := authenticatorfactory.DelegatingAuthenticatorConfig{ + Anonymous: &apiserver.AnonymousAuthConfig{ + Enabled: false, + }, + RequestHeaderConfig: requestHeaderConfig, + } + + authenticator, _, err := cfg.New() + if err != nil { + return nil, err + } + + return &DefaultAuthenticator{ + requestHeaderConfig: requestHeaderConfig, + authenticator: authenticator, + }, nil +} + +// AuthenticateRequest implements [authenticator.Request] +func (b *DefaultAuthenticator) AuthenticateRequest(req *http.Request) (*authenticator.Response, bool, error) { + return b.authenticator.AuthenticateRequest(req) +} + +// AuthenticateRequest implements [dynamiccertificates.Notifier] +// This is part of the [dynamiccertificates.CAContentProvider] interface. +func (b *DefaultAuthenticator) AddListener(listener dynamiccertificates.Listener) { + b.requestHeaderConfig.CAContentProvider.AddListener(listener) +} + +// AuthenticateRequest implements [dynamiccertificates.CAContentProvider] +func (b *DefaultAuthenticator) Name() string { + return b.requestHeaderConfig.CAContentProvider.Name() +} + +// AuthenticateRequest implements [dynamiccertificates.CAContentProvider] +func (b *DefaultAuthenticator) CurrentCABundleContent() []byte { + return b.requestHeaderConfig.CAContentProvider.CurrentCABundleContent() +} + +// AuthenticateRequest implements [dynamiccertificates.CAContentProvider] +func (b *DefaultAuthenticator) VerifyOptions() (x509.VerifyOptions, bool) { + return b.requestHeaderConfig.CAContentProvider.VerifyOptions() +} + +// AuthenticateRequest implements [dynamiccertificates.ControllerRunner] +func (b *DefaultAuthenticator) RunOnce(ctx context.Context) error { + runner, ok := b.requestHeaderConfig.CAContentProvider.(dynamiccertificates.ControllerRunner) + if !ok { + return nil + } + return runner.RunOnce(ctx) +} + +// AuthenticateRequest implements [dynamiccertificates.ControllerRunner]. +// +// It will be called by the "SecureServing" when starting the extension API server +func (b *DefaultAuthenticator) Run(ctx context.Context, workers int) { + runner, ok := b.requestHeaderConfig.CAContentProvider.(dynamiccertificates.ControllerRunner) + if !ok { + return + } + + runner.Run(ctx, workers) +} + +// Copied from https://github.com/kubernetes/apiserver/blob/v0.30.1/pkg/server/options/authentication.go#L407 +func createRequestHeaderConfig(client kubernetes.Interface) (*authenticatorfactory.RequestHeaderConfig, error) { + dynamicRequestHeaderProvider, err := newDynamicRequestHeaderController(client) + if err != nil { + return nil, fmt.Errorf("unable to create request header authentication config: %v", err) + } + + return &authenticatorfactory.RequestHeaderConfig{ + CAContentProvider: dynamicRequestHeaderProvider, + UsernameHeaders: headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.UsernameHeaders)), + GroupHeaders: headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.GroupHeaders)), + ExtraHeaderPrefixes: headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.ExtraHeaderPrefixes)), + AllowedClientNames: headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.AllowedClientNames)), + }, nil +} + +// Copied from https://github.com/kubernetes/apiserver/blob/v0.30.1/pkg/server/options/authentication_dynamic_request_header.go#L42 +func newDynamicRequestHeaderController(client kubernetes.Interface) (*options.DynamicRequestHeaderController, error) { + requestHeaderCAController, err := dynamiccertificates.NewDynamicCAFromConfigMapController( + "client-ca", + authenticationConfigMapNamespace, + authenticationConfigMapName, + "requestheader-client-ca-file", + client) + if err != nil { + return nil, fmt.Errorf("unable to create DynamicCAFromConfigMap controller: %v", err) + } + + requestHeaderAuthRequestController := headerrequest.NewRequestHeaderAuthRequestController( + authenticationConfigMapName, + authenticationConfigMapNamespace, + client, + "requestheader-username-headers", + "requestheader-group-headers", + "requestheader-extra-headers-prefix", + "requestheader-allowed-names", + ) + return &options.DynamicRequestHeaderController{ + ConfigMapCAController: requestHeaderCAController, + RequestHeaderAuthRequestController: requestHeaderAuthRequestController, + }, nil +} diff --git a/pkg/ext/apiserver_authentication_test.go b/pkg/ext/apiserver_authentication_test.go index f6649fa3c..e5b1b4515 100644 --- a/pkg/ext/apiserver_authentication_test.go +++ b/pkg/ext/apiserver_authentication_test.go @@ -1,6 +1,8 @@ package ext import ( + "context" + "crypto/tls" "encoding/json" "fmt" "io" @@ -10,6 +12,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -168,3 +171,269 @@ func TestAuthenticationCustom(t *testing.T) { }) } } + +func (s *ExtensionAPIServerSuite) TestAuthenticationDefault() { + t := s.T() + + // Same CA but CN not in the list allowed + notAllowedCertPair, err := s.ca.NewClientCert("system:not-allowed") + require.NoError(t, err) + notAllowedCert, notAllowedKey, err := notAllowedCertPair.AsBytes() + require.NoError(t, err) + + badCA, err := NewTinyCA() + require.NoError(t, err) + badCertPair, err := badCA.NewClientCert("system:auth-proxy") + require.NoError(t, err) + badCert, badKey, err := badCertPair.AsBytes() + require.NoError(t, err) + + cert, key, err := s.cert.AsBytes() + require.NoError(t, err) + certificate, err := tls.X509KeyPair(cert, key) + require.NoError(t, err) + + badCACertificate, err := tls.X509KeyPair(badCert, badKey) + require.NoError(t, err) + + notAllowedCertificate, err := tls.X509KeyPair(notAllowedCert, notAllowedKey) + require.NoError(t, err) + + scheme := runtime.NewScheme() + AddToScheme(scheme) + + store := &authnTestStore{ + testStore: &testStore{}, + userCh: make(chan user.Info, 100), + } + defaultAuth, err := NewDefaultAuthenticator(s.client) + require.NoError(t, err) + + func() { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + err = defaultAuth.RunOnce(ctx) + require.NoError(t, err) + }() + + ln, port, err := options.CreateListener("", ":0", net.ListenConfig{}) + require.NoError(t, err) + + _, cleanup, err := setupExtensionAPIServer(t, scheme, &TestType{}, &TestTypeList{}, store, func(opts *ExtensionAPIServerOptions) { + opts.Listener = ln + opts.Authenticator = defaultAuth + opts.Authorizer = authorizer.AuthorizerFunc(authzAllowAll) + }, nil) + require.NoError(t, err) + defer cleanup() + + allPaths := []string{ + "/", + "/openapi/v2", + "/openapi/v3", + "/openapi/v3/apis/ext.cattle.io/v1", + "/apis", + "/apis/ext.cattle.io", + "/apis/ext.cattle.io/v1", + "/apis/ext.cattle.io/v1/testtypes", + "/apis/ext.cattle.io/v1/testtypes/foo", + } + + type test struct { + name string + certs []tls.Certificate + paths []string + user string + groups []string + + expectedStatusCode int + expectedUser *user.DefaultInfo + } + tests := []test{ + { + name: "authenticated request check user", + certs: []tls.Certificate{certificate}, + paths: []string{"/apis/ext.cattle.io/v1/testtypes"}, + user: "my-user", + groups: []string{"my-group"}, + + expectedStatusCode: http.StatusOK, + expectedUser: &user.DefaultInfo{Name: "my-user", Groups: []string{"my-group", "system:authenticated"}, Extra: map[string][]string{}}, + }, + { + name: "authenticated request all paths", + certs: []tls.Certificate{certificate}, + paths: allPaths, + user: "my-user", + groups: []string{"my-group"}, + + expectedStatusCode: http.StatusOK, + }, + { + name: "authenticated request to unknown endpoint", + certs: []tls.Certificate{certificate}, + paths: []string{"/unknown"}, + user: "my-user", + groups: []string{"my-group"}, + + expectedStatusCode: http.StatusNotFound, + }, + { + name: "no client certs", + paths: append(allPaths, "/unknown"), + user: "my-user", + groups: []string{"my-group"}, + + expectedStatusCode: http.StatusUnauthorized, + }, + { + name: "client certs from bad CA", + certs: []tls.Certificate{badCACertificate}, + paths: append(allPaths, "/unknown"), + user: "my-user", + groups: []string{"my-group"}, + + expectedStatusCode: http.StatusUnauthorized, + }, + { + name: "client certs with CN not allowed", + certs: []tls.Certificate{notAllowedCertificate}, + paths: append(allPaths, "/unknown"), + user: "my-user", + groups: []string{"my-group"}, + + expectedStatusCode: http.StatusUnauthorized, + }, + { + name: "no user", + paths: append(allPaths, "/unknown"), + groups: []string{"my-group"}, + + expectedStatusCode: http.StatusUnauthorized, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + httpClient := http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + Certificates: test.certs, + }, + }, + } + + for _, path := range test.paths { + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://127.0.0.1:%d%s", port, path), nil) + require.NoError(t, err) + if test.user != "" { + req.Header.Set("X-Remote-User", test.user) + } + for _, group := range test.groups { + req.Header.Add("X-Remote-Group", group) + } + + // Eventually because the cache for auth might not be synced yet + require.EventuallyWithT(t, func(c *assert.CollectT) { + resp, err := httpClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + require.Equal(t, test.expectedStatusCode, resp.StatusCode) + }, 5*time.Second, 110*time.Millisecond) + + if test.expectedUser != nil { + authUser, found := store.getUser() + require.True(t, found) + require.Equal(t, test.expectedUser, authUser) + } + } + }) + } +} + +func (s *ExtensionAPIServerSuite) TestAuthenticationUnion() { + t := s.T() + + scheme := runtime.NewScheme() + AddToScheme(scheme) + + cert, key, err := s.cert.AsBytes() + require.NoError(t, err) + certificate, err := tls.X509KeyPair(cert, key) + require.NoError(t, err) + + defaultAuth, err := NewDefaultAuthenticator(s.client) + require.NoError(t, err) + + customAuth := authenticator.RequestFunc(func(req *http.Request) (*authenticator.Response, bool, error) { + user, ok := request.UserFrom(req.Context()) + if !ok { + return nil, false, nil + } + if user.GetName() == "error" { + return nil, false, fmt.Errorf("fake error") + } + return &authenticator.Response{ + User: user, + }, true, nil + }) + auth := NewUnionAuthenticator(customAuth, defaultAuth) + func() { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + err = auth.RunOnce(ctx) + require.NoError(t, err) + }() + + ln, port, err := options.CreateListener("", ":0", net.ListenConfig{}) + require.NoError(t, err) + + store := &authnTestStore{ + testStore: &testStore{}, + userCh: make(chan user.Info, 100), + } + extensionAPIServer, cleanup, err := setupExtensionAPIServer(t, scheme, &TestType{}, &TestTypeList{}, store, func(opts *ExtensionAPIServerOptions) { + opts.Listener = ln + opts.Authorizer = authorizer.AuthorizerFunc(authzAllowAll) + opts.Authenticator = auth + }, nil) + require.NoError(t, err) + defer cleanup() + + httpClient := http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + Certificates: []tls.Certificate{certificate}, + }, + }, + } + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://127.0.0.1:%d%s", port, "/openapi/v2"), nil) + require.NoError(t, err) + + userInfo := &user.DefaultInfo{ + Name: "my-user", + Groups: []string{"my-group"}, + } + + req.Header.Set("X-Remote-User", userInfo.GetName()) + req.Header.Add("X-Remote-Group", userInfo.GetGroups()[0]) + require.EventuallyWithT(t, func(c *assert.CollectT) { + resp, err := httpClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + require.Equal(t, http.StatusOK, resp.StatusCode) + }, 5*time.Second, 110*time.Millisecond) + + req = httptest.NewRequest(http.MethodGet, "/openapi/v2", nil) + w := httptest.NewRecorder() + + ctx := request.WithUser(req.Context(), userInfo) + req = req.WithContext(ctx) + + extensionAPIServer.ServeHTTP(w, req) + resp := w.Result() + require.Equal(t, http.StatusOK, resp.StatusCode) +} diff --git a/pkg/ext/apiserver_suite_test.go b/pkg/ext/apiserver_suite_test.go index 9f7cba866..5ba699200 100644 --- a/pkg/ext/apiserver_suite_test.go +++ b/pkg/ext/apiserver_suite_test.go @@ -2,14 +2,158 @@ package ext import ( "context" + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + crand "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "os" + "path/filepath" "testing" + "time" "github.com/stretchr/testify/suite" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + certutil "k8s.io/client-go/util/cert" "sigs.k8s.io/controller-runtime/pkg/envtest" ) +// Copied and modified from envtest internal +var ( + ellipticCurve = elliptic.P256() + bigOne = big.NewInt(1) +) + +// CertPair is a private key and certificate for use for client auth, as a CA, or serving. +type CertPair struct { + Key crypto.Signer + Cert *x509.Certificate +} + +// CertBytes returns the PEM-encoded version of the certificate for this pair. +func (k CertPair) CertBytes() []byte { + return pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: k.Cert.Raw, + }) +} + +// AsBytes encodes keypair in the appropriate formats for on-disk storage (PEM and +// PKCS8, respectively). +func (k CertPair) AsBytes() (cert []byte, key []byte, err error) { + cert = k.CertBytes() + + rawKeyData, err := x509.MarshalPKCS8PrivateKey(k.Key) + if err != nil { + return nil, nil, fmt.Errorf("unable to encode private key: %w", err) + } + + key = pem.EncodeToMemory(&pem.Block{ + Type: "PRIVATE KEY", + Bytes: rawKeyData, + }) + + return cert, key, nil +} + +// TinyCA supports signing serving certs and client-certs, +// and can be used as an auth mechanism with envtest. +type TinyCA struct { + CA CertPair + orgName string + + nextSerial *big.Int +} + +// newPrivateKey generates a new private key of a relatively sane size (see +// rsaKeySize). +func newPrivateKey() (crypto.Signer, error) { + return ecdsa.GenerateKey(ellipticCurve, crand.Reader) +} + +// NewTinyCA creates a new a tiny CA utility for provisioning serving certs and client certs FOR TESTING ONLY. +// Don't use this for anything else! +func NewTinyCA() (*TinyCA, error) { + caPrivateKey, err := newPrivateKey() + if err != nil { + return nil, fmt.Errorf("unable to generate private key for CA: %w", err) + } + caCfg := certutil.Config{CommonName: "envtest-environment", Organization: []string{"envtest"}} + caCert, err := certutil.NewSelfSignedCACert(caCfg, caPrivateKey) + if err != nil { + return nil, fmt.Errorf("unable to generate certificate for CA: %w", err) + } + + return &TinyCA{ + CA: CertPair{Key: caPrivateKey, Cert: caCert}, + orgName: "envtest", + nextSerial: big.NewInt(1), + }, nil +} + +func (c *TinyCA) CertBytes() []byte { + return pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: c.CA.Cert.Raw, + }) +} + +func (c *TinyCA) NewClientCert(name string) (CertPair, error) { + return c.makeCert(certutil.Config{ + CommonName: name, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }) +} + +func (c *TinyCA) makeCert(cfg certutil.Config) (CertPair, error) { + now := time.Now() + + key, err := newPrivateKey() + if err != nil { + return CertPair{}, fmt.Errorf("unable to create private key: %w", err) + } + + serial := new(big.Int).Set(c.nextSerial) + c.nextSerial.Add(c.nextSerial, bigOne) + + template := x509.Certificate{ + Subject: pkix.Name{CommonName: cfg.CommonName, Organization: cfg.Organization}, + DNSNames: cfg.AltNames.DNSNames, + IPAddresses: cfg.AltNames.IPs, + SerialNumber: serial, + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: cfg.Usages, + + // technically not necessary for testing, but let's set anyway just in case. + NotBefore: now.UTC(), + // 1 week -- the default for cfssl, and just long enough for a + // long-term test, but not too long that anyone would try to use this + // seriously. + NotAfter: now.Add(168 * time.Hour).UTC(), + } + + certRaw, err := x509.CreateCertificate(crand.Reader, &template, c.CA.Cert, key.Public(), c.CA.Key) + if err != nil { + return CertPair{}, fmt.Errorf("unable to create certificate: %w", err) + } + + cert, err := x509.ParseCertificate(certRaw) + if err != nil { + return CertPair{}, fmt.Errorf("generated invalid certificate, could not parse: %w", err) + } + + return CertPair{ + Key: key, + Cert: cert, + }, nil +} + type ExtensionAPIServerSuite struct { suite.Suite @@ -19,11 +163,43 @@ type ExtensionAPIServerSuite struct { testEnv envtest.Environment client *kubernetes.Clientset restConfig *rest.Config + + certTempPath string + ca *TinyCA + cert CertPair } func (s *ExtensionAPIServerSuite) SetupSuite() { var err error + s.ca, err = NewTinyCA() + s.Require().NoError(err) + s.cert, err = s.ca.NewClientCert("system:auth-proxy") + s.Require().NoError(err) + + cert, key, err := s.cert.AsBytes() + s.Require().NoError(err) + + s.certTempPath, err = os.MkdirTemp("", "steve_test") + s.Require().NoError(err) + + caFilepath := filepath.Join(s.certTempPath, "request-header-ca.crt") + certFilepath := filepath.Join(s.certTempPath, "client-auth-proxy.crt") + keyFilepath := filepath.Join(s.certTempPath, "client-auth-proxy.key") + + os.WriteFile(caFilepath, s.ca.CertBytes(), 0644) + os.WriteFile(certFilepath, cert, 0644) + os.WriteFile(keyFilepath, key, 0644) + + // Configures the aggregation layer according to + // https://kubernetes.io/docs/tasks/extend-kubernetes/configure-aggregation-layer/#enable-kubernetes-apiserver-flags apiServer := &envtest.APIServer{} + apiServer.Configure().Append("requestheader-allowed-names", "system:auth-proxy") + apiServer.Configure().Append("requestheader-extra-headers-prefix", "X-Remote-Extra-") + apiServer.Configure().Append("requestheader-group-headers", "X-Remote-Group") + apiServer.Configure().Append("requestheader-username-headers", "X-Remote-User") + apiServer.Configure().Append("requestheader-client-ca-file", caFilepath) + apiServer.Configure().Append("proxy-client-cert-file", certFilepath) + apiServer.Configure().Append("proxy-client-key-file", keyFilepath) s.testEnv = envtest.Environment{ ControlPlane: envtest.ControlPlane{ @@ -43,6 +219,7 @@ func (s *ExtensionAPIServerSuite) TearDownSuite() { s.cancel() err := s.testEnv.Stop() s.Require().NoError(err) + os.RemoveAll(s.certTempPath) } func TestExtensionAPIServerSuite(t *testing.T) {