feat: Add short CLI param to config-file

* Move Port to Web param group
* Add ENV option to Port
* Remove default value from config-file

doc: Update readme

* Add parameters to README

Add LoadConfigFromYaml tests
* db_port as uint16
* remove parameter from LoadConfigFromYaml method

refactor: Use config.DBConfig as parameter in db.Init

Sync interval configuration

doc: Update README for sync-interval config
* Use sync-interval in retry sleep too

Author: AndresCidoncha

README.md

@@ -78,6 +78,38 @@ requires:
   you need to make sure the provided AWS credentials have `dynamodb:Scan` access to that
   table.
 
+## Configuration
+
+Terraboard currently supports configuration in three different ways:
+
+1. Environment variables
+2. CLI parameters
+3. Configuration file (YAML). A configuration file example can be found in the root directory of this repository.
+
+The precedence of configurations is as described below.
+
+### Available parameters
+
+|CLI|ENV|YAML|Description|Default|
+|---|---|----|-----------|-------|
+|`-V | --version`| - | - | Prints version | - |
+|`-p | --port`|`PORT`|`web.port`|Port to listen on| 8080 |
+|`-c | --config-file`|`CONFIG_FILE`|-|Config File path| - |
+|`-l | --log-level` | `TERRABOARD_LOG_LEVEL` | `log.level` | Set log level (debug, info, warn, error, fatal, panic) | info |
+|`--log-format` | `TERRABOARD_LOG_FORMAT` | `log.format` | Set log format (plain, json) | plain |
+|`--db-host` | `DB_HOST` | `db.host` | Database host | db |
+|`--db-port` | `DB_PORT` | `db.port` | Database port | 5432 |
+|`--db-user` | `DB_USER` | `db.user` | Database user | gorm |
+|`--db-password` | `DB_PASSWORD` | `db.password` | Database password | - |
+|`--db-name` | `DB_NAME` | `db.name` | Database name | gorm |
+|`--no-sync` | - | `db.no-sync` | Do not sync database | false |
+|`--sync-interval` | - | `db.sync-interval` | DB sync interval (in minutes) | 1 |
+|`--dynamodb-table` | `AWS_DYNAMODB_TABLE` | `aws.dynamodb-table` | AWS DynamoDB table for locks | - |
+|`--s3-bucket` | `AWS_BUCKET` | `aws.bucket` | AWS S3 bucket | - |
+|`--key-prefix` | `AWS_KEY_PREFIX` | `aws.key-prefix` | AWS Key Prefix | - |
+|`--file-extension` | `AWS_FILE_EXTENSION` | `aws.file-extension` | File extension of state files | .tfstate |
+|`--base-url` | `TERRABOARD_BASE_URL` | `web.base-url` | Base URL | / |
+|`--logout-url` | `TERRABOARD_LOGOUT_URL` | `web.logout-url` | Logout URL | - |
 
 ## Use with Docker
 
@@ -111,6 +143,8 @@ Then point your browser to http://localhost:8080.
 
 To use the included compose file, you will need to configure an [OAuth application](https://developer.github.com/apps/building-oauth-apps/).
 
+Configuration file can be provided to the container using a [volume](https://docs.docker.com/compose/compose-file/#volumes) or a [configuration](https://docs.docker.com/compose/compose-file/#configs).
+
 ## Use with Rancher
 
 [Camptocamp's Rancher Catalog](https://github.com/camptocamp/camptocamp-rancher-catalog)

config/config.go

@@ -11,48 +11,62 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
+// LogConfig stores the log configuration
+type LogConfig struct {
+	Level  string `short:"l" long:"log-level" env:"TERRABOARD_LOG_LEVEL" yaml:"level" description:"Set log level ('debug', 'info', 'warn', 'error', 'fatal', 'panic')." default:"info"`
+	Format string `long:"log-format" yaml:"format" env:"TERRABOARD_LOG_FORMAT" description:"Set log format ('plain', 'json')." default:"plain"`
+}
+
+// DBConfig stores the database configuration
+type DBConfig struct {
+	Host         string `long:"db-host" env:"DB_HOST" yaml:"host" description:"Database host." default:"db"`
+	Port         uint16 `long:"db-port" env:"DB_PORT" yaml:"port" description:"Database port." default:"5432"`
+	User         string `long:"db-user" env:"DB_USER" yaml:"user" description:"Database user." default:"gorm"`
+	Password     string `long:"db-password" env:"DB_PASSWORD" yaml:"password" description:"Database password."`
+	Name         string `long:"db-name" env:"DB_NAME" yaml:"name" description:"Database name." default:"gorm"`
+	NoSync       bool   `long:"no-sync" yaml:"no-sync" description:"Do not sync database."`
+	SyncInterval uint16 `long:"sync-interval" yaml:"sync-interval" description:"DB sync interval (in minutes)" default:"1"`
+}
+
+// S3BucketConfig stores the S3 bucket configuration
+type S3BucketConfig struct {
+	Bucket        string `long:"s3-bucket" env:"AWS_BUCKET" yaml:"bucket" description:"AWS S3 bucket."`
+	KeyPrefix     string `long:"key-prefix" env:"AWS_KEY_PREFIX" yaml:"key-prefix" description:"AWS Key Prefix."`
+	FileExtension string `long:"file-extension" env:"AWS_FILE_EXTENSION" yaml:"file-extension" description:"File extension of state files." default:".tfstate"`
+}
+
+// AWSConfig stores the DynamoDB table and S3 Bucket configuration
+type AWSConfig struct {
+	DynamoDBTable string         `long:"dynamodb-table" env:"AWS_DYNAMODB_TABLE" yaml:"dynamodb-table" description:"AWS DynamoDB table for locks."`
+	S3            S3BucketConfig `group:"S3 Options" yaml:"s3"`
+}
+
+// WebConfig stores the UI interface parameters
+type WebConfig struct {
+	Port      uint16 `short:"p" long:"port" env:"TERRABOARD_PORT" yaml:"port" description:"Port to listen on." default:"8080"`
+	BaseURL   string `long:"base-url" env:"TERRABOARD_BASE_URL" yaml:"base-url" description:"Base URL." default:"/"`
+	LogoutURL string `long:"logout-url" env:"TERRABOARD_LOGOUT_URL" yaml:"logout-url" description:"Logout URL."`
+}
+
 // Config stores the handler's configuration and UI interface parameters
 type Config struct {
 	Version bool `short:"V" long:"version" description:"Display version."`
 
-	Port int `short:"p" long:"port" yaml:"port" description:"Port to listen on." default:"8080"`
-
-	ConfigFilePath string `long:"config-file" env:"CONFIG_FILE" description:"Config File path" default:""`
-
-	Log struct {
-		Level  string `short:"l" long:"log-level" yaml:"level" description:"Set log level ('debug', 'info', 'warn', 'error', 'fatal', 'panic')." env:"TERRABOARD_LOG_LEVEL" default:"info"`
-		Format string `long:"log-format" yaml:"format" description:"Set log format ('plain', 'json')." env:"TERRABOARD_LOG_FORMAT" default:"plain"`
-	} `group:"Logging Options" yaml:"log"`
-
-	DB struct {
-		Host     string `long:"db-host" env:"DB_HOST" yaml:"host" description:"Database host." default:"db"`
-		Port     string `long:"db-port" env:"DB_PORT" yaml:"port" description:"Database port." default:"5432"`
-		User     string `long:"db-user" env:"DB_USER" yaml:"user" description:"Database user." default:"gorm"`
-		Password string `long:"db-password" env:"DB_PASSWORD" yaml:"password" description:"Database password."`
-		Name     string `long:"db-name" env:"DB_NAME" yaml:"name" description:"Database name." default:"gorm"`
-		NoSync   bool   `long:"no-sync" yaml:"no-sync" description:"Do not sync database."`
-	} `group:"Database Options" yaml:"database"`
-
-	AWS struct {
-		DynamoDBTable string `long:"dynamodb-table" env:"AWS_DYNAMODB_TABLE" yaml:"dynamodb-table" description:"AWS DynamoDB table for locks."`
-
-		S3 struct {
-			Bucket        string `long:"s3-bucket" env:"AWS_BUCKET" yaml:"bucket" description:"AWS S3 bucket."`
-			KeyPrefix     string `long:"key-prefix" env:"AWS_KEY_PREFIX" yaml:"key-prefix" description:"AWS Key Prefix."`
-			FileExtension string `long:"file-extension" env:"AWS_FILE_EXTENSION" yaml:"file-extension" description:"File extension of state files." default:".tfstate"`
-		} `group:"S3 Options" yaml:"s3"`
-	} `group:"AWS Options" yaml:"aws"`
-
-	Web struct {
-		BaseURL   string `long:"base-url" env:"TERRABOARD_BASE_URL" yaml:"base-url" description:"Base URL." default:"/"`
-		LogoutURL string `long:"logout-url" env:"TERRABOARD_LOGOUT_URL" yaml:"logout-url" description:"Logout URL."`
-	} `group:"Web" yaml:"web"`
+	ConfigFilePath string `short:"c" long:"config-file" env:"CONFIG_FILE" description:"Config File path"`
+
+	Log LogConfig `group:"Logging Options" yaml:"log"`
+
+	DB DBConfig `group:"Database Options" yaml:"database"`
+
+	AWS AWSConfig `group:"AWS Options" yaml:"aws"`
+
+	Web WebConfig `group:"Web" yaml:"web"`
 }
 
 // LoadConfigFromYaml loads the config from config file
-func (c *Config) LoadConfigFromYaml(configFilePath string) *Config {
+func (c *Config) LoadConfigFromYaml() *Config {
 	fmt.Printf("Loading config from %s\n", c.ConfigFilePath)
-	yamlFile, err := ioutil.ReadFile(configFilePath)
+	yamlFile, err := ioutil.ReadFile(c.ConfigFilePath)
 	if err != nil {
 		log.Printf("yamlFile.Get err #%v ", err)
 	}
@@ -75,7 +89,7 @@ func LoadConfig(version string) *Config {
 
 	if c.ConfigFilePath != "" {
 		if _, err := os.Stat(c.ConfigFilePath); err == nil {
-			c.LoadConfigFromYaml(c.ConfigFilePath)
+			c.LoadConfigFromYaml()
 		} else {
 			fmt.Printf("File %s doesn't exists!\n", c.ConfigFilePath)
 			os.Exit(1)

config/config_test.go

@@ -1,10 +1,46 @@
 package config
 
 import (
+	"testing"
+
 	log "github.com/sirupsen/logrus"
 )
 
-import "testing"
+func TestLoadConfigFromYaml(t *testing.T) {
+	compareConfig := Config{
+		Log: LogConfig{
+			Level:  "error",
+			Format: "json",
+		},
+		ConfigFilePath: "config_test.yml",
+		DB: DBConfig{
+			Host:     "postgres",
+			Port:     15432,
+			User:     "terraboard-user",
+			Password: "terraboard-pass",
+			Name:     "terraboard-db",
+			NoSync:   true,
+		},
+		AWS: AWSConfig{
+			DynamoDBTable: "terraboard-dynamodb",
+			S3: S3BucketConfig{
+				Bucket:        "terraboard-bucket",
+				KeyPrefix:     "test/",
+				FileExtension: ".tfstate",
+			},
+		},
+		Web: WebConfig{
+			Port:      39090,
+			BaseURL:   "/test/",
+			LogoutURL: "/test-logout",
+		},
+	}
+	c := Config{ConfigFilePath: "config_test.yml"}
+	c.LoadConfigFromYaml()
+	if c != compareConfig {
+		t.Fatalf("Expected: %v\nGot: %v", compareConfig, c)
+	}
+}
 
 func TestSetLogging_debug(t *testing.T) {
 	c := Config{}

config/config_test.yml

@@ -0,0 +1,23 @@
+log:
+  level: error
+  format: json
+
+database:
+  host: postgres
+  port: 15432
+  user: terraboard-user
+  password: terraboard-pass
+  name: terraboard-db
+  no-sync: true
+
+aws:
+  dynamodb-table: terraboard-dynamodb
+  s3:
+    bucket: terraboard-bucket
+    key-prefix: test/
+    file-extension: .tfstate
+
+web:
+  port: 39090
+  base-url: /test/
+  logout-url: /test-logout

db/db.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/camptocamp/terraboard/config"
 	"github.com/camptocamp/terraboard/types"
 	"github.com/hashicorp/terraform/states"
 	"github.com/hashicorp/terraform/states/statefile"
@@ -26,9 +27,9 @@ type Database struct {
 var pageSize = 20
 
 // Init setups up the Database and a pointer to it
-func Init(host, port, user, dbname, password, logLevel string) *Database {
+func Init(config config.DBConfig, debug bool) *Database {
 	var err error
-	connString := fmt.Sprintf("host=%s port=%s user=%s dbname=%s sslmode=disable password=%s", host, port, user, dbname, password)
+	connString := fmt.Sprintf("host=%s port=%d user=%s dbname=%s sslmode=disable password=%s", config.Host, config.Port, config.User, config.Name, config.Password)
 	db, err := gorm.Open("postgres", connString)
 	if err != nil {
 		log.Fatal(err)
@@ -37,7 +38,7 @@ func Init(host, port, user, dbname, password, logLevel string) *Database {
 	log.Infof("Automigrate")
 	db.AutoMigrate(&types.Version{}, &types.State{}, &types.Module{}, &types.Resource{}, &types.Attribute{})
 
-	if logLevel == "debug" {
+	if debug {
 		db.LogMode(true)
 	}
 	return &Database{db}

example.yml

@@ -1,5 +1,3 @@
-port: 9090
-
 log:
   level: info
   format: plain
@@ -10,7 +8,8 @@ database:
   user: terraboard
   password: terraboard
   name: terraboard
-  no-sync: true
+  no-sync: false
+  sync-interval: 5
 
 aws:
   dynamodb-table: terraboard
@@ -20,5 +19,6 @@ aws:
     file-extension: .tfstate
 
 web:
+  port: 9090
   base-url: /terraboard/
   logout-url: /test

main.go

@@ -54,15 +54,16 @@ func isKnownStateVersion(statesVersions map[string][]string, versionID, path str
 
 // Refresh the DB from S3
 // This should be the only direct bridge between S3 and the DB
-func refreshDB(d *db.Database) {
+func refreshDB(syncInterval uint16, d *db.Database) {
+	interval := time.Duration(syncInterval) * time.Minute
 	for {
 		log.Infof("Refreshing DB from S3")
 		states, err := s3.GetStates()
 		if err != nil {
 			log.WithFields(log.Fields{
 				"error": err,
 			}).Error("Failed to retrieve states from S3. Retrying in 1 minute.")
-			time.Sleep(1 * time.Minute)
+			time.Sleep(interval)
 			continue
 		}
 
@@ -105,7 +106,8 @@ func refreshDB(d *db.Database) {
 			}
 		}
 
-		time.Sleep(1 * time.Minute)
+		log.Debugf("Waiting %d minutes until next DB sync", syncInterval)
+		time.Sleep(interval)
 	}
 }
 
@@ -145,14 +147,11 @@ func main() {
 	auth.Setup(c)
 
 	// Set up the DB and start S3->DB sync
-	database := db.Init(
-		c.DB.Host, c.DB.Port,
-		c.DB.User, c.DB.Password,
-		c.DB.Name, c.Log.Level)
+	database := db.Init(c.DB, c.Log.Level == "debug")
 	if c.DB.NoSync {
 		log.Infof("Not syncing database, as requested.")
 	} else {
-		go refreshDB(database)
+		go refreshDB(c.DB.SyncInterval, database)
 	}
 	defer database.Close()
 
@@ -181,5 +180,6 @@ func main() {
 	http.HandleFunc(util.GetFullPath("api/tf_versions"), handleWithDB(api.ListTfVersions, database))
 
 	// Start server
-	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%v", c.Port), nil))
+	log.Debugf("Listening on port %d\n", c.Web.Port)
+	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%v", c.Web.Port), nil))
 }