🤫 WIP: Secret experiment

.buildkite/secrets-pipeline-example.yml

@@ -0,0 +1,223 @@
+env:
+  DRY_RUN: false # set to true to disable publishing releases
+agents:
+  queue: agent-runners-linux-amd64
+
+secret_providers:
+  - id: ssm
+    type: aws-ssm
+      iam_role: arn:aws:iam::123456789012:role/buildkite-role
+      assume_via_oidc: true
+
+  - id: ssm-in-another-account
+    type: aws-ssm
+    config:
+      iam_role: arn:aws:iam::555555555555:role/buildkite-role-2
+      assume_via_oidc: true
+
+  - id: vault
+    type: vault_kvv2
+    config:
+      address: https://vault.example.com
+      mount_path: secret
+      auth_type: kubernetes
+      auth_config:
+        role: buildkite
+        service_account_token_path: /var/run/secrets/kubernetes.io/serviceaccount/token
+
+secrets:
+  - env_var: AWS_NEPTUNE_LOGIN_PASSWORD
+    key: path/to/neptune_login_password
+    provider_id: ssm
+
+  - env_var: WIDGET_CORP_API_KEY
+    key: path/to/widget-corp-api-key
+    provider_id: vault
+
+  - env_var: GADGET_CO_TOKEN
+    key: path/to/gadget-co-token
+    provider_id: ssm-in-another-account
+
+  - file: ~/.ssh/id_rsa
+    key: path/to/ssh-key
+    provider_id: ssm
+
+steps:
+  - name: ":go: go fmt"
+    key: test-go-fmt
+    command: ".buildkite/steps/test-go-fmt.sh"
+    plugins:
+      docker-compose#v3.0.0:
+        config: .buildkite/docker-compose.yml
+        run: agent
+
+  - name: ":linux: Linux AMD64 Tests"
+    key: test-linux-amd64
+    command: ".buildkite/steps/tests.sh"
+    artifact_paths: junit-*.xml
+    plugins:
+      docker-compose#v3.0.0:
+        config: .buildkite/docker-compose.yml
+        run: agent
+
+  - name: ":linux: Linux ARM64 Tests"
+    key: test-linux-arm64
+    command: ".buildkite/steps/tests.sh"
+    artifact_paths: junit-*.xml
+    agents:
+      queue: agent-runners-linux-arm64
+    plugins:
+      docker-compose#v3.0.0:
+        config: .buildkite/docker-compose.yml
+        run: agent
+
+  - name: ":satellite: Detect Data Races"
+    key: test-race-linux-arm64
+    command: ".buildkite/steps/tests.sh -race"
+    artifact_paths: junit-*.xml
+    agents:
+      queue: agent-runners-linux-arm64
+    plugins:
+      docker-compose#v3.0.0:
+        config: .buildkite/docker-compose.yml
+        run: agent
+
+  - name: ":windows: Windows AMD64 Tests"
+    key: test-windows
+    command: "bash .buildkite\\steps\\tests.sh"
+    artifact_paths: junit-*.xml
+    agents:
+      queue: agent-runners-windows-amd64
+
+  - label: ":writing_hand: Annotate with Test Failures"
+    depends_on:
+      - test-linux-amd64
+      - test-race-linux-arm64
+      - test-linux-arm64
+      - test-windows
+    plugins:
+      - junit-annotate#v1.6.0:
+          artifacts: junit-*.xml
+
+  - group: ":hammer_and_wrench: Binary builds"
+    steps:
+    - name: ":{{matrix.os}}: Build {{matrix.os}} {{matrix.arch}} binary"
+      command: ".buildkite/steps/build-binary.sh {{matrix.os}} {{matrix.arch}}"
+      key: build-binary
+      depends_on:
+        # don't wait for slower windows tests
+        - test-linux-amd64
+        - test-linux-arm64
+      artifact_paths: "pkg/*"
+      plugins:
+        docker-compose#v3.0.0:
+          config: .buildkite/docker-compose.yml
+          run: agent
+      matrix:
+        setup:
+          os:
+            - darwin
+            - freebsd
+            - linux
+            - openbsd
+            - windows
+          arch:
+            - "386"
+            - amd64
+            - arm64
+        adjustments:
+          - with: { os: darwin, arch: "386" }
+            skip: "macOS no longer supports x86 binaries"
+
+          - with: { os: dragonflybsd, arch: amd64 }
+
+          - with: { os: freebsd, arch: arm64 }
+            skip: "arm64 FreeBSD is not currently supported"
+
+          - with: { os: linux, arch: arm }
+          - with: { os: linux, arch: armhf }
+          - with: { os: linux, arch: ppc64 }
+          - with: { os: linux, arch: ppc64le }
+          - with: { os: linux, arch: mips64le }
+          - with: { os: linux, arch: s390x }
+
+          - with: { os: netbsd, arch: amd64 }
+
+          - with: { os: openbsd, arch: arm64 }
+            skip: "arm64 OpenBSD is not currently supported"
+
+  - name: ":technologist: Test bk cli + Agent cli"
+    key: test-bk-cli
+    depends_on: build-binary
+    command: ".buildkite/steps/test-bk.sh"
+    plugins:
+      docker-compose#v3.0.0:
+        config: .buildkite/docker-compose.yml
+        run: agent
+        env:
+          - BUILDKITE_AGENT_ACCESS_TOKEN
+          - BUILDKITE_BUILD_ID
+          - BUILDKITE_JOB_ID
+        volumes:
+          - "/usr/bin/buildkite-agent:/usr/bin/buildkite-agent"
+
+  - name: ":mag: Extract Agent Version Metadata"
+    key: set-metadata
+    command: ".buildkite/steps/extract-agent-version-metadata.sh"
+
+  - group: ":docker: Docker Container Builds"
+    steps:
+      - name: ":docker: {{matrix}} container build"
+        key: build-docker
+        agents:
+          queue: elastic-builders
+        depends_on:
+          - build-binary
+          - set-metadata
+        command: ".buildkite/steps/build-docker-image.sh {{matrix}}"
+        matrix:
+          setup:
+            - "alpine"
+            - "alpine-k8s"
+            - "ubuntu-18.04"
+            - "ubuntu-20.04"
+            - "sidecar"
+
+  - name: ":debian: Debian package build"
+    key: build-debian-packages
+    depends_on:
+      - build-binary
+      - set-metadata
+    command: ".buildkite/steps/build-debian-packages.sh"
+    artifact_paths: "deb/**/*"
+
+  - name: ":redhat: RPM Package build"
+    key: build-rpm-packages
+    depends_on:
+      - build-binary
+      - set-metadata
+    command: ".buildkite/steps/build-rpm-packages.sh"
+    artifact_paths: "rpm/**/*"
+
+  - name: ":github: Build Github Release"
+    key: build-github-release
+    depends_on:
+      - build-binary
+      - set-metadata
+    command: ".buildkite/steps/build-github-release.sh"
+    artifact_paths: "releases/**/*"
+    plugins:
+      docker-compose#v1.8.0:
+        config: .buildkite/docker-compose.release.yml
+        run: github-release
+
+  - name: ":pipeline: Upload Release Pipeline"
+    key: upload-release-steps
+    depends_on:
+      - test-windows
+      - test-bk-cli
+      - build-rpm-packages
+      - build-debian-packages
+      - build-docker
+      - build-github-release
+    command: ".buildkite/steps/upload-release-steps.sh"

bootstrap/bootstrap.go

@@ -23,6 +23,7 @@ import (
 	"github.com/buildkite/agent/v3/hook"
 	"github.com/buildkite/agent/v3/process"
 	"github.com/buildkite/agent/v3/redaction"
+	"github.com/buildkite/agent/v3/secrets"
 	"github.com/buildkite/agent/v3/tracetools"
 	"github.com/buildkite/agent/v3/utils"
 	"github.com/buildkite/roko"
@@ -113,6 +114,90 @@ func (b *Bootstrap) Run(ctx context.Context) (exitCode int) {
 		return shell.GetExitCode(err)
 	}
 
+	b.shell.Headerf("🤫 Fetching Build Secrets")
+
+	// Just pretend that these two jsons live in the pipeline.yml, and get passed through as env vars by the backend
+	secretProviderRegistryJSON := `[
+  {
+    "id": "ssm",
+    "type": "aws-ssm",
+    "config": {}
+  },
+  {
+    "id": "other-ssm",
+    "type": "aws-ssm",
+    "config": {
+      "role_arn": "arn:aws:iam::555555555555:role/benno-test-role-delete-after-2022-11-29"
+    }
+  }
+]`
+
+	secretProviderRegistry, err := secrets.NewProviderRegistryFromJSON(secrets.ProviderRegistryConfig{Shell: b.shell}, secretProviderRegistryJSON)
+	if err != nil {
+		b.shell.Errorf("Error creating secret provider registry: %v", err)
+		return 1
+	}
+
+	secretsJSON := []byte(`[
+  {
+    "env_var": "SUPER_SECRET_ENV_VAR",
+    "key": "/benno/secret/envar",
+    "provider_id": "ssm"
+  },
+  {
+    "file": "/Users/ben/secret-file",
+    "key": "/benno/secret/file",
+    "provider_id": "other-ssm"
+  }
+]`)
+
+	var secretConfigs []secrets.SecretConfig
+	err = json.Unmarshal(secretsJSON, &secretConfigs)
+	if err != nil {
+		b.shell.Errorf("Error unmarshalling secrets: %v", err)
+		return 1
+	}
+
+	validationErrs := make([]error, 0, len(secretConfigs))
+	for _, c := range secretConfigs {
+		err := c.Validate()
+		if err != nil {
+			validationErrs = append(validationErrs, err)
+		}
+	}
+
+	for _, err := range validationErrs {
+		b.shell.Errorf("Error validating secret config: %v", err)
+	}
+
+	if len(validationErrs) > 0 {
+		return 1
+	}
+
+	fetchedSecrets, errors := secretProviderRegistry.FetchAll(secretConfigs)
+	if len(errors) > 0 {
+		b.shell.Errorf("Errors fetching secrets:")
+		for _, err := range errors {
+			b.shell.Errorf("  %v", err)
+		}
+		return 1
+	}
+
+	for _, secret := range fetchedSecrets {
+		// TODO: Automatically add env secrets to the redactor
+		err := secret.Store()
+		if err != nil {
+			b.shell.Errorf("Error storing secret: %v", err)
+		}
+
+		defer func(secret secrets.Secret) {
+			err := secret.Cleanup()
+			if err != nil {
+				b.shell.Warningf("Error cleaning up secret: %s", err)
+			}
+		}(secret)
+	}
+
 	var includePhase = func(phase string) bool {
 		if len(b.Phases) == 0 {
 			return true

go.mod

@@ -75,6 +75,7 @@ require (
 	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
 	github.com/philhofer/fwd v1.1.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/puzpuzpuz/xsync v1.5.2
 	github.com/qri-io/jsonpointer v0.0.0-20180309164927-168dd9e45cf2 // indirect
 	github.com/russross/blackfriday/v2 v2.0.1 // indirect
 	github.com/sasha-s/go-deadlock v0.0.0-20180226215254-237a9547c8a5 // indirect

go.sum

@@ -228,6 +228,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/puzpuzpuz/xsync v1.5.2 h1:yRAP4wqSOZG+/4pxJ08fPTwrfL0IzE/LKQ/cw509qGY=
+github.com/puzpuzpuz/xsync v1.5.2/go.mod h1:K98BYhX3k1dQ2M63t1YNVDanbwUPmBCAhNmVrrxfiGg=
 github.com/qri-io/jsonpointer v0.0.0-20180309164927-168dd9e45cf2 h1:C8RRfIlExwwrXw28G8LkrpWiHUVT4uLowfvjUYJ2Iec=
 github.com/qri-io/jsonpointer v0.0.0-20180309164927-168dd9e45cf2/go.mod h1:DnJPaYgiKu56EuDp8TU5wFLdZIcAnb/uH9v37ZaMV64=
 github.com/qri-io/jsonschema v0.0.0-20180607150648-d0d3b10ec792 h1:vwTGeGWCew89DI4ZwKCaobGAN7ExvZiBzgn4LZHMVOc=

secrets/provider.go

@@ -0,0 +1,37 @@
+package secrets
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+type providerCandidate struct {
+	Type   string          `json:"type"`
+	ID     string          `json:"id"`
+	Config json.RawMessage `json:"config"`
+}
+
+func (r providerCandidate) Initialize() (Provider, error) {
+	switch r.Type {
+	case "aws-ssm":
+		var conf AWSSSMProviderConfig
+		err := json.Unmarshal(r.Config, &conf)
+		if err != nil {
+			return nil, fmt.Errorf("unmarshalling config for aws-ssm provider %s: %v", r.ID, err)
+		}
+
+		ssm, err := NewAWSSSMProvider(r.ID, conf)
+		if err != nil {
+			return nil, fmt.Errorf("creating aws-ssm provider %s: %w", r.ID, err)
+		}
+
+		return ssm, nil
+	default:
+		return nil, fmt.Errorf("invalid provider type %s for provider %s", r.Type, r.ID)
+	}
+}
+
+// A Provider is a source of secrets, and must provide a way to fetch a secret given some key. Providers must be goroutine-safe.
+type Provider interface {
+	Fetch(key string) (string, error)
+}