Implemented support for Terraform Enterprise as state provider

Author: mvisonneau

api/api.go

@@ -134,9 +134,9 @@ func StateCompare(w http.ResponseWriter, r *http.Request, d *db.Database) {
 }
 
 // GetLocks returns information on locked States
-func GetLocks(w http.ResponseWriter, r *http.Request) {
+func GetLocks(w http.ResponseWriter, r *http.Request, sp state.Provider) {
 	w.Header().Set("Access-Control-Allow-Origin", "*")
-	locks, err := state.GetLocks()
+	locks, err := sp.GetLocks()
 	if err != nil {
 		JSONError(w, "Failed to get locks", err)
 		return

config/config.go

@@ -43,7 +43,9 @@ type AWSConfig struct {
 
 // TFEConfig stores the Terraform Enterprise configuration
 type TFEConfig struct {
-	Token string         `long:"tfe-token" env:"TFE_TOKEN" yaml:"tfe-token" description:"Terraform Enterprise Token for state access"`
+	Address      string `long:"tfe-address" env:"TFE_ADDRESS" yaml:"tfe-address" description:"Terraform Enterprise address for states access"`
+	Token        string `long:"tfe-token" env:"TFE_TOKEN" yaml:"tfe-token" description:"Terraform Enterprise Token for states access"`
+	Organization string `long:"tfe-organization" env:"TFE_ORGANIZATION" yaml:"tfe-organization" description:"Terraform Enterprise organization for states access"`
 }
 
 // WebConfig stores the UI interface parameters

go.mod

@@ -5,6 +5,7 @@ go 1.13
 require (
 	github.com/aws/aws-sdk-go v1.26.6
 	github.com/denisenkom/go-mssqldb v0.0.0-20190820223206-44cdfe8d8ba9 // indirect
+	github.com/hashicorp/go-tfe v0.3.27
 	github.com/hashicorp/hcl/v2 v2.2.0 // indirect
 	github.com/hashicorp/hil v0.0.0-20190212132231-97b3a9cdfa93 // indirect
 	github.com/hashicorp/terraform v0.12.18

go.sum

@@ -121,6 +121,7 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -145,6 +146,7 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv
 github.com/hashicorp/go-azure-helpers v0.10.0/go.mod h1:YuAtHxm2v74s+IjQwUG88dHBJPd5jL+cXr5BGVzSKhE=
 github.com/hashicorp/go-checkpoint v0.5.0/go.mod h1:7nfLNL10NsxqO4iWuW6tWW0HjZuDrwkBuEQsVcpCOgg=
 github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-getter v1.4.0/go.mod h1:7qxyCd8rBfcShwsvxgIguu4KbS3l8bUCwg2Umn7RjeY=
 github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI=
@@ -155,11 +157,14 @@ github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0/go.mod h1:
 github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-plugin v1.0.1-0.20190610192547-a1bc61569a26/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY=
+github.com/hashicorp/go-retryablehttp v0.5.2 h1:AoISa4P4IsW0/m4T6St8Yw38gTl5GtBAgfkhYh1xAz4=
 github.com/hashicorp/go-retryablehttp v0.5.2/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
 github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I=
+github.com/hashicorp/go-slug v0.4.1 h1:/jAo8dNuLgSImoLXaX7Od7QB4TfYCVPam+OpAt5bZqc=
 github.com/hashicorp/go-slug v0.4.1/go.mod h1:I5tq5Lv0E2xcNXNkmx7BSfzi1PsJ2cNjs3cC3LwyhK8=
 github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
+github.com/hashicorp/go-tfe v0.3.27 h1:7XZ/ZoPyYoeuNXaWWW0mJOq016y0qb7I4Q0P/cagyu8=
 github.com/hashicorp/go-tfe v0.3.27/go.mod h1:DVPSW2ogH+M9W1/i50ASgMht8cHP7NxxK0nrY9aFikQ=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
@@ -314,6 +319,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/svanharmelen/jsonapi v0.0.0-20180618144545-0c0828c3f16d h1:Z4EH+5EffvBEhh37F0C0DnpklTMh00JOkjW5zK3ofBI=
 github.com/svanharmelen/jsonapi v0.0.0-20180618144545-0c0828c3f16d/go.mod h1:BSTlc8jOjh0niykqEGVXOLXdi9o0r0kR8tCYiMvjFgw=
 github.com/terraform-providers/terraform-provider-openstack v1.15.0/go.mod h1:2aQ6n/BtChAl1y2S60vebhyJyZXBsuAI5G4+lHrT1Ew=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -416,6 +422,7 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

main.go

@@ -41,6 +41,12 @@ func handleWithDB(apiF func(w http.ResponseWriter, r *http.Request, d *db.Databa
 	})
 }
 
+func handleWithStateProvider(apiF func(w http.ResponseWriter, r *http.Request, sp state.Provider), sp state.Provider) func(http.ResponseWriter, *http.Request) {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		apiF(w, r, sp)
+	})
+}
+
 func isKnownStateVersion(statesVersions map[string][]string, versionID, path string) bool {
 	if v, ok := statesVersions[versionID]; ok {
 		for _, s := range v {
@@ -52,24 +58,24 @@ func isKnownStateVersion(statesVersions map[string][]string, versionID, path str
 	return false
 }
 
-// Refresh the DB from S3
-// This should be the only direct bridge between S3 and the DB
-func refreshDB(syncInterval uint16, d *db.Database) {
+// Refresh the DB
+// This should be the only direct bridge between the state provider and the DB
+func refreshDB(syncInterval uint16, d *db.Database, sp state.Provider) {
 	interval := time.Duration(syncInterval) * time.Minute
 	for {
-		log.Infof("Refreshing DB from S3")
-		states, err := state.GetStates()
+		log.Infof("Refreshing DB")
+		states, err := sp.GetStates()
 		if err != nil {
 			log.WithFields(log.Fields{
 				"error": err,
-			}).Error("Failed to retrieve states from S3. Retrying in 1 minute.")
+			}).Error("Failed to retrieve states. Retrying in 1 minute.")
 			time.Sleep(interval)
 			continue
 		}
 
 		statesVersions := d.ListStatesVersions()
 		for _, st := range states {
-			versions, _ := state.GetVersions(st)
+			versions, _ := sp.GetVersions(st)
 			for _, v := range versions {
 				if _, ok := statesVersions[v.ID]; ok {
 					log.WithFields(log.Fields{
@@ -86,7 +92,7 @@ func refreshDB(syncInterval uint16, d *db.Database) {
 					}).Debug("State is already in the database, skipping")
 					continue
 				}
-				state, err := state.GetState(st, v.ID)
+				state, err := sp.GetState(st, v.ID)
 				if err != nil {
 					log.WithFields(log.Fields{
 						"path":       st,
@@ -141,7 +147,10 @@ func main() {
 	}
 
 	// Set up the state provider
-	state.Configure(c)
+	sp, err := state.Configure(c)
+	if err != nil {
+		log.Fatal(err)
+	}
 
 	// Set up auth
 	auth.Setup(c)
@@ -151,7 +160,7 @@ func main() {
 	if c.DB.NoSync {
 		log.Infof("Not syncing database, as requested.")
 	} else {
-		go refreshDB(c.DB.SyncInterval, database)
+		go refreshDB(c.DB.SyncInterval, database, sp)
 	}
 	defer database.Close()
 
@@ -171,7 +180,7 @@ func main() {
 	http.HandleFunc(util.GetFullPath("api/state/"), handleWithDB(api.GetState, database))
 	http.HandleFunc(util.GetFullPath("api/state/activity/"), handleWithDB(api.GetStateActivity, database))
 	http.HandleFunc(util.GetFullPath("api/state/compare/"), handleWithDB(api.StateCompare, database))
-	http.HandleFunc(util.GetFullPath("api/locks"), api.GetLocks)
+	http.HandleFunc(util.GetFullPath("api/locks"), handleWithStateProvider(api.GetLocks, sp))
 	http.HandleFunc(util.GetFullPath("api/search/attribute"), handleWithDB(api.SearchAttribute, database))
 	http.HandleFunc(util.GetFullPath("api/resource/types"), handleWithDB(api.ListResourceTypes, database))
 	http.HandleFunc(util.GetFullPath("api/resource/types/count"), handleWithDB(api.ListResourceTypesWithCount, database))

state/aws.go

@@ -18,25 +18,25 @@ import (
 
 // AWS is a state provider type, leveraging S3 and DynamoDB
 type AWS struct {
-	svc *s3.S3
-	dynamoSvc *dynamodb.DynamoDB
-	bucket string
-	dynamoTable string
-	keyPrefix string
+	svc           *s3.S3
+	dynamoSvc     *dynamodb.DynamoDB
+	bucket        string
+	dynamoTable   string
+	keyPrefix     string
 	fileExtension string
 }
 
 // NewAWS creates an AWS object
 func NewAWS(c *config.Config) AWS {
 	sess := session.Must(session.NewSession())
-	
+
 	return AWS{
-		svc: s3.New(sess, &aws_sdk.Config{}),
-		bucket: c.AWS.S3.Bucket,
-		keyPrefix: c.AWS.S3.KeyPrefix,
+		svc:           s3.New(sess, &aws_sdk.Config{}),
+		bucket:        c.AWS.S3.Bucket,
+		keyPrefix:     c.AWS.S3.KeyPrefix,
 		fileExtension: c.AWS.S3.FileExtension,
-		dynamoSvc: dynamodb.New(sess, &aws_sdk.Config{}),
-		dynamoTable: c.AWS.DynamoDBTable,
+		dynamoSvc:     dynamodb.New(sess, &aws_sdk.Config{}),
+		dynamoTable:   c.AWS.DynamoDBTable,
 	}
 }
 
@@ -146,7 +146,7 @@ func (a *AWS) GetVersions(state string) (versions []Version, err error) {
 
 	for _, v := range result.Versions {
 		versions = append(versions, Version{
-			ID: *v.VersionId,
+			ID:           *v.VersionId,
 			LastModified: *v.LastModified,
 		})
 	}

state/state.go

@@ -27,7 +27,7 @@ type Lock struct {
 
 // Version is a handler for state versions
 type Version struct {
-	ID string
+	ID           string
 	LastModified time.Time
 }
 
@@ -39,30 +39,18 @@ type Provider interface {
 	GetState(string, string) (*statefile.File, error)
 }
 
-// Provider is a handler for the configured one
-var p Provider
+// Configure the state provider
+func Configure(c *config.Config) (Provider, error) {
+	if len(c.TFE.Token) > 0 {
+		log.Info("Using Terraform Enterprise as the state/locks provider")
+		provider, err := NewTFE(c)
+		if err != nil {
+			return nil, err
+		}
+		return &provider, nil
+	}
 
-// Configure returns the configured provider
-func Configure(c *config.Config) {
 	log.Info("Using AWS (S3+DynamoDB) as the state/locks provider")
 	provider := NewAWS(c)
-	p = &provider
-}
-
-// Functions wrappers
-//
-func GetLocks() (map[string]LockInfo, error) {
-	return p.GetLocks()
-}
-
-func GetVersions(state string) ([]Version, error) {
-	return p.GetVersions(state)
-}
-
-func GetStates() ([]string, error) {
-	return p.GetStates()
-}
-
-func GetState(state, versionID string) (*statefile.File, error) {
-	return p.GetState(state, versionID)
+	return &provider, nil
 }

state/state_test.go

@@ -1,3 +1,3 @@
 package state
 
-// TODO: write!
+// TODO: write!