Add terraform and manual installation support (#4246)

This tweaks the BYOC cloud installation methods to enable manual installation; and installation via terraform. Example commands: To install automatically via CloudFormation: `earthly cloud install --via cloudformation --name <stack-name>` To install automatically via Terraform: `earthly cloud install --via terraform` To install manually: ``` earthly cloud install \ --name <name> \ --aws-account-id <aws-account-id> \ --aws-region <aws-region> \ --aws-security-group-id <aws-security-group-id> \ --aws-ssh-key-id <aws-ssh-key-id> \ --aws-subnet-id <aws-subnet-id> \ --aws-instance-profile-arn <aws-instance-profile-arn> \ --aws-earthly-access-role-arn <aws-earthly-access-role-arn> ``` Docs forthcoming on the new installation options. --------- Co-authored-by: Brandon Schurman <[email protected]>
earthly · Jul 12, 2024 · cb38f72 · cb38f72
1 parent 39086ad
commit cb38f72
Show file tree

Hide file tree

Showing 4 changed files with 200 additions and 22 deletions.
diff --git a/cloud/cloud_installation.go b/cloud/cloud_installation.go
@@ -14,6 +14,11 @@ const (
 	CloudStatusUnknown = "Unknown"
 )
 
+const (
+	AddressResolutionPrivateIP  = "private_ip"
+	AddressResolutionPrivateDNS = "private_dns"
+)
+
 type Installation struct {
 	Name          string
 	Org           string
@@ -33,9 +38,14 @@ type CloudConfigurationOpt struct {
 	SecurityGroupId    string
 	Region             string
 	InstanceProfileArn string
+	AddressResolution  string
 }
 
 func (c *Client) ConfigureCloud(ctx context.Context, orgID string, configuration *CloudConfigurationOpt) (*Installation, error) {
+	resolution := pb.AddressResolution_ADDRESS_RESOLUTION_PRIVATE_IP
+	if configuration.AddressResolution == AddressResolutionPrivateDNS {
+		resolution = pb.AddressResolution_ADDRESS_RESOLUTION_PRIVATE_DNS
+	}
 	resp, err := c.compute.ConfigureCloud(c.withAuth(ctx), &pb.ConfigureCloudRequest{
 		OrgId:              orgID,
 		Name:               configuration.Name,
@@ -47,6 +57,7 @@ func (c *Client) ConfigureCloud(ctx context.Context, orgID string, configuration
 		SecurityGroupId:    configuration.SecurityGroupId,
 		Region:             configuration.Region,
 		InstanceProfileArn: configuration.InstanceProfileArn,
+		AddressResolution:  resolution,
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "error from ConfigureCloud API")

diff --git a/cmd/earthly/subcmd/cloud_installation_cmds.go b/cmd/earthly/subcmd/cloud_installation_cmds.go
@@ -2,14 +2,16 @@ package subcmd
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
-	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/fatih/color"
 	"os"
+	"os/exec"
 	"text/tabwriter"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
 	awsconfig "github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/service/cloudformation"
+	"github.com/fatih/color"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
 
@@ -19,8 +21,25 @@ import (
 
 type CloudInstallation struct {
 	cli CLI
+
+	method            string
+	cloudName         string
+	addressResolution string
+
+	awsAccountID          string
+	awsRegion             string
+	awsSecurityGroup      string
+	awsSSHKeyName         string
+	awsSubnetID           string
+	awsInstanceProfileARN string
+	awsEarthlyRoleARN     string
 }
 
+const (
+	cloudInstallationMethodCloudFormation = "cloudformation"
+	cloudInstallationMethodTerraform      = "terraform"
+)
+
 func NewCloudInstallation(cli CLI) *CloudInstallation {
 	return &CloudInstallation{
 		cli: cli,
@@ -51,6 +70,58 @@ func (a *CloudInstallation) Cmds() []*cli.Command {
 					Description: "Configure a new Cloud Installation.",
 					UsageText:   "earthly cloud install <cloud-name>",
 					Action:      a.install,
+					Flags: []cli.Flag{
+						&cli.StringFlag{
+							Name:        "name",
+							Usage:       "The name of the cloud installation",
+							Destination: &a.cloudName,
+						},
+						&cli.StringFlag{
+							Name:        "via",
+							Usage:       "The source to use for automatic cloud installation. Valid options are (cloudformation, terraform). See docs for details.",
+							Destination: &a.method,
+						},
+						&cli.StringFlag{
+							Name:        "resolution",
+							Usage:       "How to connect to the remote satellite in your cloud. Depends on your individual setup. Valid options are (private_dns, private_ip). See docs for details.",
+							Destination: &a.addressResolution,
+						},
+						&cli.StringFlag{
+							Name:        "aws-account-id",
+							Usage:       "The account ID for the AWS account the BYOC installation belongs to.",
+							Destination: &a.awsAccountID,
+						},
+						&cli.StringFlag{
+							Name:        "aws-region",
+							Usage:       "The region for the AWS account the BYOC installation belongs to.",
+							Destination: &a.awsRegion,
+						},
+						&cli.StringFlag{
+							Name:        "aws-security-group-id",
+							Usage:       "The ID for the security group the BYOC installation will launch new satellites into.",
+							Destination: &a.awsSecurityGroup,
+						},
+						&cli.StringFlag{
+							Name:        "aws-ssh-key-name",
+							Usage:       "The name of the ssh key for BYOC to include on each newly launched satellite. Useful for debugging.",
+							Destination: &a.awsSSHKeyName,
+						},
+						&cli.StringFlag{
+							Name:        "aws-subnet-id",
+							Usage:       "The ID of the subnet the BYOC installation will launch new satellites into.",
+							Destination: &a.awsSubnetID,
+						},
+						&cli.StringFlag{
+							Name:        "aws-instance-profile-arn",
+							Usage:       "The ARN of the IAM instance profile for the BYOC installation to use for new satellites.",
+							Destination: &a.awsInstanceProfileARN,
+						},
+						&cli.StringFlag{
+							Name:        "aws-earthly-access-role-arn",
+							Usage:       "The ARN of the IAM role for Earthly to use when orchestrating and managing the new BYOC satellites.",
+							Destination: &a.awsEarthlyRoleARN,
+						},
+					},
 				},
 				{
 					Name:        "use",
@@ -84,14 +155,17 @@ func (c *CloudInstallation) install(cliCtx *cli.Context) error {
 	c.cli.SetCommandName("installCloud")
 	ctx := cliCtx.Context
 
-	if cliCtx.NArg() == 0 {
-		return errors.New("cloud name is required")
-	}
-	if cliCtx.NArg() > 1 {
-		return errors.New("only a single cloud name is supported")
+	if cliCtx.NArg() > 0 {
+		return errors.New("invalid number of arguments provided")
 	}
 
-	cloudName := cliCtx.Args().Get(0)
+	if !cliCtx.IsSet("resolution") {
+		c.addressResolution = cloud.AddressResolutionPrivateIP
+	}
+	if c.addressResolution != cloud.AddressResolutionPrivateIP &&
+		c.addressResolution != cloud.AddressResolutionPrivateDNS {
+		return errors.Errorf("%q is not a valid value for --resolution", c.addressResolution)
+	}
 
 	cloudClient, err := helper.NewCloudClient(c.cli)
 	if err != nil {
@@ -103,14 +177,24 @@ func (c *CloudInstallation) install(cliCtx *cli.Context) error {
 		return err
 	}
 
-	installation, err := c.getInstallationDataFromCloudFormation(ctx, cloudName)
+	var installationOpt *cloud.CloudConfigurationOpt
+	switch {
+	case c.method == cloudInstallationMethodCloudFormation:
+		installationOpt, err = c.getInstallationDataFromCloudFormation(ctx)
+	case c.method == cloudInstallationMethodTerraform:
+		installationOpt, err = c.getInstallationDataFromTerraform(ctx)
+	case c.isManualInstallation(cliCtx):
+		installationOpt, err = c.manualInstallation(ctx)
+	default:
+		return errors.New("could not determine installation method")
+	}
 	if err != nil {
 		return err
 	}
 
-	c.cli.Console().Printf("Configuring new Cloud Installation: %s. Please wait...", cloudName)
+	c.cli.Console().Printf("Configuring new Cloud Installation: %s. Please wait...", installationOpt.Name)
 
-	install, err := cloudClient.ConfigureCloud(ctx, orgID, installation)
+	install, err := cloudClient.ConfigureCloud(ctx, orgID, installationOpt)
 	if err != nil {
 		return errors.Wrap(err, "failed installing cloud")
 	}
@@ -125,7 +209,7 @@ func (c *CloudInstallation) install(cliCtx *cli.Context) error {
 	c.cli.Console().Printf("Cloud Installation was successful. Current status of cloud is: %s", install.Status)
 	c.cli.Console().Printf("")
 	c.cli.Console().Printf("To make your new cloud the default destination for future satellite launches, run the following:")
-	c.cli.Console().Printf("  earthly cloud use %s", cloudName)
+	c.cli.Console().Printf("  earthly cloud use %s", installationOpt.Name)
 	c.cli.Console().Printf("")
 
 	return nil
@@ -265,7 +349,75 @@ func (c *CloudInstallation) printTable(installations []cloud.Installation) {
 	}
 }
 
-func (c *CloudInstallation) getInstallationDataFromCloudFormation(ctx context.Context, stackName string) (*cloud.CloudConfigurationOpt, error) {
+func (c *CloudInstallation) manualInstallation(_ context.Context) (*cloud.CloudConfigurationOpt, error) {
+	return &cloud.CloudConfigurationOpt{
+		Name:               c.cloudName,
+		SshKeyName:         c.awsSSHKeyName,
+		ComputeRoleArn:     c.awsEarthlyRoleARN,
+		AccountId:          c.awsAccountID,
+		AllowedSubnetIds:   []string{c.awsSubnetID},
+		SecurityGroupId:    c.awsSecurityGroup,
+		Region:             c.awsRegion,
+		InstanceProfileArn: c.awsInstanceProfileARN,
+		AddressResolution:  c.addressResolution,
+	}, nil
+}
+
+func (c *CloudInstallation) getInstallationDataFromTerraform(ctx context.Context) (*cloud.CloudConfigurationOpt, error) {
+	// assumes you are authed with aws however you need to be for terraform to work
+	cmd := exec.CommandContext(ctx, "terraform", "output", "-json")
+	cmdOutput, err := cmd.CombinedOutput()
+	if err != nil {
+		return nil, fmt.Errorf("could not get terraform outputs: %s: %w", string(cmdOutput), err)
+	}
+
+	tfOutputs := make(map[string]json.RawMessage)
+	err = json.Unmarshal(cmdOutput, &tfOutputs)
+	if err != nil {
+		return nil, fmt.Errorf("could not parse terraform outputs: %w", err)
+	}
+
+	data, ok := tfOutputs[c.cloudName]
+	if !ok {
+		return nil, fmt.Errorf("could not find output %s", c.cloudName)
+	}
+
+	installOutputs := struct {
+		Value map[string]string `json:"value"`
+	}{}
+	err = json.Unmarshal(data, &installOutputs)
+	if err != nil {
+		return nil, fmt.Errorf("could not find output value for key %s", c.cloudName)
+	}
+
+	configOpt := &cloud.CloudConfigurationOpt{
+		AddressResolution: c.addressResolution,
+	}
+	for k, v := range installOutputs.Value {
+		switch k {
+		case "account_id":
+			configOpt.AccountId = v
+		case "allowed_subnet_id":
+			configOpt.AllowedSubnetIds = []string{v}
+		case "compute_role_arn":
+			configOpt.ComputeRoleArn = v
+		case "installation_name":
+			configOpt.Name = v
+		case "instance_profile_arn":
+			configOpt.InstanceProfileArn = v
+		case "region":
+			configOpt.Region = v
+		case "security_group_id":
+			configOpt.SecurityGroupId = v
+		case "ssh_key_name":
+			configOpt.SshKeyName = v
+		}
+	}
+
+	return configOpt, nil
+}
+
+func (c *CloudInstallation) getInstallationDataFromCloudFormation(ctx context.Context) (*cloud.CloudConfigurationOpt, error) {
 	awsConfig, err := awsconfig.LoadDefaultConfig(ctx)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not load aws config")
@@ -274,25 +426,27 @@ func (c *CloudInstallation) getInstallationDataFromCloudFormation(ctx context.Co
 	client := cloudformation.NewFromConfig(awsConfig)
 
 	describeStacksOutput, err := client.DescribeStacks(ctx, &cloudformation.DescribeStacksInput{
-		StackName: aws.String(stackName),
+		StackName: aws.String(c.cloudName),
 	})
 	if err != nil {
-		return nil, errors.Wrap(err, fmt.Sprintf("could not describe stack %s", stackName))
+		return nil, errors.Wrap(err, fmt.Sprintf("could not describe stack %s", c.cloudName))
 	}
 
 	if len(describeStacksOutput.Stacks) != 1 {
-		return nil, fmt.Errorf("unexpected number of stacks(%v) found with name %q", len(describeStacksOutput.Stacks), stackName)
+		return nil, fmt.Errorf("unexpected number of stacks(%v) found with name %q", len(describeStacksOutput.Stacks), c.cloudName)
 	}
 
 	stack := describeStacksOutput.Stacks[0]
-	params := &cloud.CloudConfigurationOpt{}
+	params := &cloud.CloudConfigurationOpt{
+		AddressResolution: c.addressResolution,
+	}
 
 	for _, output := range stack.Outputs {
 		if output.OutputKey == nil {
-			return nil, fmt.Errorf("specified stack %s has nil output key", stackName)
+			return nil, fmt.Errorf("specified stack %s has nil output key", c.cloudName)
 		}
 		if output.OutputValue == nil {
-			return nil, fmt.Errorf("specified stack %s has nil value for key %s", stackName, *output.OutputKey)
+			return nil, fmt.Errorf("specified stack %s has nil value for key %s", c.cloudName, *output.OutputKey)
 		}
 
 		switch *output.OutputKey {
@@ -317,3 +471,16 @@ func (c *CloudInstallation) getInstallationDataFromCloudFormation(ctx context.Co
 
 	return params, nil
 }
+
+func (c *CloudInstallation) isManualInstallation(ctx *cli.Context) bool {
+	// If any aws arg is specified, then
+	// how to see if flag is set instead of value here
+
+	return ctx.IsSet("aws-account-id") &&
+		ctx.IsSet("aws-region") &&
+		ctx.IsSet("aws-security-group-id") &&
+		ctx.IsSet("aws-ssh-key-name") &&
+		ctx.IsSet("aws-subnet-id") &&
+		ctx.IsSet("aws-instance-profile-arn") &&
+		ctx.IsSet("aws-earthly-access-role-arn")
+}
diff --git a/go.mod b/go.mod
@@ -19,7 +19,7 @@ require (
 	github.com/docker/go-connections v0.4.0
 	github.com/docker/go-units v0.5.0
 	github.com/dustin/go-humanize v1.0.1
-	github.com/earthly/cloud-api v1.0.1-0.20240530080539-c5171a73ad6f
+	github.com/earthly/cloud-api v1.0.1-0.20240712142419-23b6f0913996
 	github.com/earthly/earthly/ast v0.0.0-00010101000000-000000000000
 	github.com/earthly/earthly/util/deltautil v0.0.0-20240507235053-335389ed3e2a
 	github.com/elastic/go-sysinfo v1.9.0

diff --git a/go.sum b/go.sum
@@ -194,8 +194,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/earthly/buildkit v0.0.0-20240515200521-531b303aa8ec h1:vf6x0fPOWKakjH3n2N1O9Tg5j1HDIJsC3Kkgmuko2U0=
 github.com/earthly/buildkit v0.0.0-20240515200521-531b303aa8ec/go.mod h1:1/yAC8A0Tu94Bdmv07gaG1pFBp+CetVwO7oB3qvZXUc=
-github.com/earthly/cloud-api v1.0.1-0.20240530080539-c5171a73ad6f h1:QjB03JW1E4dS4nvkqE1JKrVxGs95jBBqMYhM6LOEd34=
-github.com/earthly/cloud-api v1.0.1-0.20240530080539-c5171a73ad6f/go.mod h1:rU/tYJ7GFBjdKAITV2heDbez++glpGSbtJaZcp73rNI=
+github.com/earthly/cloud-api v1.0.1-0.20240712142419-23b6f0913996 h1:UobaUMrXjUcVaHXAev9aOflHHRg9gO+uOwhCvWJ9+cw=
+github.com/earthly/cloud-api v1.0.1-0.20240712142419-23b6f0913996/go.mod h1:rU/tYJ7GFBjdKAITV2heDbez++glpGSbtJaZcp73rNI=
 github.com/earthly/fsutil v0.0.0-20231030221755-644b08355b65 h1:6oyWHoxHXwcTt4EqmMw6361scIV87uEAB1N42+VpIwk=
 github.com/earthly/fsutil v0.0.0-20231030221755-644b08355b65/go.mod h1:9kMVqMyQ/Sx2df5LtnGG+nbrmiZzCS7V6gjW3oGHsvI=
 github.com/elastic/go-sysinfo v1.9.0 h1:usICqY/Nw4Mpn9f4LdtpFrKxXroJDe81GaxxUlCckIo=