Merge pull request #274 from verult/cloudconfig

Integration with GCE cloud config

cmd/main.go

@@ -33,8 +33,9 @@ func init() {
 }
 
 var (
-	endpoint      = flag.String("endpoint", "unix:/tmp/csi.sock", "CSI endpoint")
-	vendorVersion string
+	endpoint          = flag.String("endpoint", "unix:/tmp/csi.sock", "CSI endpoint")
+	gceConfigFilePath = flag.String("cloud-config", "", "Path to GCE cloud provider config")
+	vendorVersion     string
 )
 
 const (
@@ -57,7 +58,7 @@ func handle() {
 	gceDriver := driver.GetGCEDriver()
 
 	//Initialize GCE Driver (Move setup to main?)
-	cloudProvider, err := gce.CreateCloudProvider(vendorVersion)
+	cloudProvider, err := gce.CreateCloudProvider(vendorVersion, *gceConfigFilePath)
 	if err != nil {
 		glog.Fatalf("Failed to get cloud provider: %v", err)
 	}

pkg/gce-cloud-provider/compute/gce.go

@@ -15,7 +15,10 @@ limitations under the License.
 package gcecloudprovider
 
 import (
+	"context"
 	"fmt"
+	"golang.org/x/oauth2/google"
+	"gopkg.in/gcfg.v1"
 	"net/http"
 	"os"
 	"runtime"
@@ -24,9 +27,8 @@ import (
 	"cloud.google.com/go/compute/metadata"
 	"github.com/golang/glog"
 	"golang.org/x/oauth2"
-	"golang.org/x/oauth2/google"
 	beta "google.golang.org/api/compute/v0.beta"
-	compute "google.golang.org/api/compute/v1"
+	"google.golang.org/api/compute/v1"
 	"google.golang.org/api/googleapi"
 	"k8s.io/apimachinery/pkg/util/wait"
 )
@@ -57,20 +59,44 @@ type CloudProvider struct {
 
 var _ GCECompute = &CloudProvider{}
 
-func CreateCloudProvider(vendorVersion string) (*CloudProvider, error) {
-	svc, err := createCloudService(vendorVersion)
+type ConfigFile struct {
+	Global ConfigGlobal `gcfg:"global"`
+}
+
+type ConfigGlobal struct {
+	TokenURL  string `gcfg:"token-url"`
+	TokenBody string `gcfg:"token-body"`
+	ProjectId string `gcfg:"project-id"`
+}
+
+func CreateCloudProvider(vendorVersion string, configPath string) (*CloudProvider, error) {
+	configFile, err := readConfig(configPath)
+	if err != nil {
+		return nil, err
+	}
+	// At this point configFile could still be nil.
+	// Any following code that uses configFile should handle nil pointer gracefully.
+
+	glog.V(1).Infof("Using GCE provider config %+v", configFile)
+
+	tokenSource, err := generateTokenSource(configFile)
+	if err != nil {
+		return nil, err
+	}
+
+	svc, err := createCloudService(vendorVersion, tokenSource)
 	if err != nil {
 		return nil, err
 	}
 
-	betasvc, err := createBetaCloudService(vendorVersion)
+	betasvc, err := createBetaCloudService(vendorVersion, tokenSource)
 	if err != nil {
 		return nil, err
 	}
 
-	project, zone, err := getProjectAndZoneFromMetadata()
+	project, zone, err := getProjectAndZone(configFile)
 	if err != nil {
-		return nil, fmt.Errorf("Failed getting Project and Zone from Metadata server: %v", err)
+		return nil, fmt.Errorf("Failed getting Project and Zone: %v", err)
 	}
 
 	return &CloudProvider{
@@ -83,8 +109,55 @@ func CreateCloudProvider(vendorVersion string) (*CloudProvider, error) {
 
 }
 
-func createBetaCloudService(vendorVersion string) (*beta.Service, error) {
-	client, err := newDefaultOauthClient()
+func generateTokenSource(configFile *ConfigFile) (oauth2.TokenSource, error) {
+
+	if configFile != nil && configFile.Global.TokenURL != "" && configFile.Global.TokenURL != "nil" {
+		// configFile.Global.TokenURL is defined
+		// Use AltTokenSource
+
+		tokenSource := NewAltTokenSource(configFile.Global.TokenURL, configFile.Global.TokenBody)
+		glog.V(4).Infof("Using AltTokenSource %#v", tokenSource)
+		return tokenSource, nil
+	}
+
+	// Use DefaultTokenSource
+
+	tokenSource, err := google.DefaultTokenSource(
+		context.Background(),
+		compute.CloudPlatformScope,
+		compute.ComputeScope)
+
+	// DefaultTokenSource relies on GOOGLE_APPLICATION_CREDENTIALS env var being set.
+	if gac, ok := os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS"); ok {
+		glog.V(4).Infof("GOOGLE_APPLICATION_CREDENTIALS env var set %v", gac)
+	} else {
+		glog.Warningf("GOOGLE_APPLICATION_CREDENTIALS env var not set")
+	}
+	glog.V(4).Infof("Using DefaultTokenSource %#v", tokenSource)
+
+	return tokenSource, err
+}
+
+func readConfig(configPath string) (*ConfigFile, error) {
+	if configPath == "" {
+		return nil, nil
+	}
+
+	reader, err := os.Open(configPath)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't open cloud provider configuration at %s: %v", configPath, err)
+	}
+	defer reader.Close()
+
+	cfg := &ConfigFile{}
+	if err := gcfg.FatalOnly(gcfg.ReadInto(cfg, reader)); err != nil {
+		return nil, fmt.Errorf("couldn't read cloud provider configuration at %s: %v", configPath, err)
+	}
+	return cfg, nil
+}
+
+func createBetaCloudService(vendorVersion string, tokenSource oauth2.TokenSource) (*beta.Service, error) {
+	client, err := newOauthClient(tokenSource)
 	if err != nil {
 		return nil, err
 	}
@@ -96,13 +169,13 @@ func createBetaCloudService(vendorVersion string) (*beta.Service, error) {
 	return service, nil
 }
 
-func createCloudService(vendorVersion string) (*compute.Service, error) {
-	svc, err := createCloudServiceWithDefaultServiceAccount(vendorVersion)
+func createCloudService(vendorVersion string, tokenSource oauth2.TokenSource) (*compute.Service, error) {
+	svc, err := createCloudServiceWithDefaultServiceAccount(vendorVersion, tokenSource)
 	return svc, err
 }
 
-func createCloudServiceWithDefaultServiceAccount(vendorVersion string) (*compute.Service, error) {
-	client, err := newDefaultOauthClient()
+func createCloudServiceWithDefaultServiceAccount(vendorVersion string, tokenSource oauth2.TokenSource) (*compute.Service, error) {
+	client, err := newOauthClient(tokenSource)
 	if err != nil {
 		return nil, err
 	}
@@ -114,22 +187,7 @@ func createCloudServiceWithDefaultServiceAccount(vendorVersion string) (*compute
 	return service, nil
 }
 
-func newDefaultOauthClient() (*http.Client, error) {
-	// No compute token source, fallback on default
-	tokenSource, err := google.DefaultTokenSource(
-		oauth2.NoContext,
-		compute.CloudPlatformScope,
-		compute.ComputeScope)
-	if gac, ok := os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS"); ok {
-		glog.V(4).Infof("GOOGLE_APPLICATION_CREDENTIALS env var set %v", gac)
-	} else {
-		glog.Warningf("GOOGLE_APPLICATION_CREDENTIALS env var not set")
-	}
-	glog.V(4).Infof("Using DefaultTokenSource %#v", tokenSource)
-	if err != nil {
-		return nil, err
-	}
-
+func newOauthClient(tokenSource oauth2.TokenSource) (*http.Client, error) {
 	if err := wait.PollImmediate(5*time.Second, 30*time.Second, func() (bool, error) {
 		if _, err := tokenSource.Token(); err != nil {
 			glog.Errorf("error fetching initial token: %v", err)
@@ -140,18 +198,32 @@ func newDefaultOauthClient() (*http.Client, error) {
 		return nil, err
 	}
 
-	return oauth2.NewClient(oauth2.NoContext, tokenSource), nil
+	return oauth2.NewClient(context.Background(), tokenSource), nil
 }
 
-func getProjectAndZoneFromMetadata() (string, string, error) {
+func getProjectAndZone(config *ConfigFile) (string, string, error) {
+	var err error
+
 	zone, err := metadata.Zone()
 	if err != nil {
 		return "", "", err
 	}
-	projectID, err := metadata.ProjectID()
-	if err != nil {
-		return "", "", err
+
+	var projectID string
+	if config == nil || config.Global.ProjectId == "" {
+		// Project ID is not available from the local GCE cloud provider config file.
+		// This could happen if the driver is not running in the master VM.
+		// Defaulting to project ID from the Metadata server.
+		projectID, err = metadata.ProjectID()
+		if err != nil {
+			return "", "", err
+		}
+		glog.V(4).Infof("Using GCP project ID from the Metadata server: %q", projectID)
+	} else {
+		projectID = config.Global.ProjectId
+		glog.V(4).Infof("Using GCP project ID from the local GCE cloud provider config file: %q", projectID)
 	}
+
 	return projectID, zone, nil
 }
 

pkg/gce-cloud-provider/compute/token_source.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gcecloudprovider
+
+import (
+	"encoding/json"
+	"net/http"
+	"strings"
+	"time"
+
+	"k8s.io/client-go/util/flowcontrol"
+
+	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/google"
+	"google.golang.org/api/googleapi"
+)
+
+const (
+	// Max QPS to allow through to the token URL.
+	tokenURLQPS = .05 // back off to once every 20 seconds when failing
+	// Maximum burst of requests to token URL before limiting.
+	tokenURLBurst = 3
+)
+
+// TODO(#276) add metrics around token requests once the driver integrates with Prometheus.
+
+// AltTokenSource is the structure holding the data for the functionality needed to generates tokens
+type AltTokenSource struct {
+	oauthClient *http.Client
+	tokenURL    string
+	tokenBody   string
+	throttle    flowcontrol.RateLimiter
+}
+
+// Token returns a token which may be used for authentication
+func (a *AltTokenSource) Token() (*oauth2.Token, error) {
+	a.throttle.Accept()
+	return a.token()
+}
+
+func (a *AltTokenSource) token() (*oauth2.Token, error) {
+	req, err := http.NewRequest("POST", a.tokenURL, strings.NewReader(a.tokenBody))
+	if err != nil {
+		return nil, err
+	}
+	res, err := a.oauthClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+	if err := googleapi.CheckResponse(res); err != nil {
+		return nil, err
+	}
+	var tok struct {
+		AccessToken string    `json:"accessToken"`
+		ExpireTime  time.Time `json:"expireTime"`
+	}
+	if err := json.NewDecoder(res.Body).Decode(&tok); err != nil {
+		return nil, err
+	}
+	return &oauth2.Token{
+		AccessToken: tok.AccessToken,
+		Expiry:      tok.ExpireTime,
+	}, nil
+}
+
+// NewAltTokenSource constructs a new alternate token source for generating tokens.
+func NewAltTokenSource(tokenURL, tokenBody string) oauth2.TokenSource {
+	client := oauth2.NewClient(oauth2.NoContext, google.ComputeTokenSource(""))
+	a := &AltTokenSource{
+		oauthClient: client,
+		tokenURL:    tokenURL,
+		tokenBody:   tokenBody,
+		throttle:    flowcontrol.NewTokenBucketRateLimiter(tokenURLQPS, tokenURLBurst),
+	}
+	return oauth2.ReuseTokenSource(nil, a)
+}