-
Notifications
You must be signed in to change notification settings - Fork 302
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
8 changed files
with
429 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| package secrets | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "fmt" | ||
| ) | ||
|
|
||
| type providerCandidate struct { | ||
| Type string `json:"type"` | ||
| ID string `json:"id"` | ||
| Config json.RawMessage `json:"config"` | ||
| } | ||
|
|
||
| func (r providerCandidate) Initialize() (Provider, error) { | ||
| switch r.Type { | ||
| case "aws-ssm": | ||
| var conf AWSSSMProviderConfig | ||
| err := json.Unmarshal(r.Config, &conf) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("unmarshalling config for aws-ssm provider %s: %v", r.ID, err) | ||
| } | ||
|
|
||
| ssm, err := NewAWSSSMProvider(r.ID, conf) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("creating aws-ssm provider %s: %w", r.ID, err) | ||
| } | ||
|
|
||
| return ssm, nil | ||
| default: | ||
| return nil, fmt.Errorf("invalid provider type %s for provider %s", r.Type, r.ID) | ||
| } | ||
| } | ||
|
|
||
| // A Provider is a source of secrets, and must provide a way to fetch a secret given some key. Providers must be goroutine-safe. | ||
| type Provider interface { | ||
| Fetch(key string) (string, error) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| package secrets | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "os" | ||
|
|
||
| "github.com/aws/aws-sdk-go/aws" | ||
| "github.com/aws/aws-sdk-go/aws/credentials" | ||
| "github.com/aws/aws-sdk-go/aws/credentials/stscreds" | ||
| "github.com/aws/aws-sdk-go/aws/session" | ||
| "github.com/aws/aws-sdk-go/service/ssm" | ||
| "github.com/aws/aws-sdk-go/service/sts" | ||
| ) | ||
|
|
||
| type AWSSSMProviderConfig struct { | ||
| ID string `json:"id"` | ||
| RoleARN string `json:"role_arn"` | ||
| AssumeViaOIDC bool `json:"assume_via_oidc"` | ||
| } | ||
|
|
||
| type AWSSSMProvider struct { | ||
| id string | ||
| ssmI *ssm.SSM | ||
| } | ||
|
|
||
| func NewAWSSSMProvider(id string, config AWSSSMProviderConfig) (*AWSSSMProvider, error) { | ||
| sess, err := session.NewSession() | ||
| if err != nil { | ||
| return nil, fmt.Errorf("initialising AWS session: %w", err) | ||
| } | ||
|
|
||
| return &AWSSSMProvider{ | ||
| id: id, | ||
| ssmI: generateSSMInstance(sess, config), | ||
| }, nil | ||
| } | ||
|
|
||
| func (s *AWSSSMProvider) ID() string { | ||
| return s.id | ||
| } | ||
|
|
||
| func (s *AWSSSMProvider) Type() string { | ||
| return "aws-ssm" | ||
| } | ||
|
|
||
| func (s *AWSSSMProvider) Fetch(key string) (string, error) { | ||
| out, err := s.ssmI.GetParameter(&ssm.GetParameterInput{ | ||
| Name: aws.String(key), | ||
| WithDecryption: aws.Bool(true), | ||
| }) | ||
|
|
||
| if err != nil { | ||
| return "", fmt.Errorf("retrieving secret %s from SSM Parameter Store: %w", key, err) | ||
| } | ||
|
|
||
| return *out.Parameter.Value, nil | ||
| } | ||
|
|
||
| func generateSSMInstance(sess *session.Session, config AWSSSMProviderConfig) *ssm.SSM { | ||
| if config.RoleARN == "" { | ||
| return ssm.New(sess) | ||
| } | ||
|
|
||
| if config.AssumeViaOIDC { | ||
| stsClient := sts.New(sess) | ||
| sessionName := fmt.Sprintf("buildkite-agent-aws-ssm-secrets-provider-%s", os.Getenv("BUILDKITE_JOB_ID")) | ||
| // TODO: Use BK OIDC provider instead of some rando file | ||
| roleProvider := stscreds.NewWebIdentityRoleProviderWithOptions(stsClient, config.RoleARN, sessionName, stscreds.FetchTokenPath("/build/token")) | ||
| creds := credentials.NewCredentials(roleProvider) | ||
| return ssm.New(sess, &aws.Config{Credentials: creds}) | ||
| } | ||
|
|
||
| creds := stscreds.NewCredentials(sess, config.RoleARN) | ||
| return ssm.New(sess, &aws.Config{Credentials: creds}) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,131 @@ | ||
| package secrets | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "fmt" | ||
| "sync" | ||
|
|
||
| "github.com/buildkite/agent/v3/bootstrap/shell" | ||
| "github.com/puzpuzpuz/xsync" | ||
| ) | ||
|
|
||
| type ProviderRegistry struct { | ||
| config ProviderRegistryConfig | ||
| // We have the two maps here because we only want to initialise a provider if a secret using that provider is used | ||
| // We shouldn't boot up any secrets providers that won't get used | ||
| candidates *xsync.MapOf[string, providerCandidate] // Candidates technically doesn't need to be a sync map as it's never altered after initialization, but i've made it one just for symmetry | ||
| providers *xsync.MapOf[string, Provider] | ||
| } | ||
|
|
||
| type ProviderRegistryConfig struct { | ||
| Shell *shell.Shell | ||
| } | ||
|
|
||
| // NewProviderRegistryFromJSON takes a JSON string representing a slice of secrets.RawProvider, and returns a ProviderRegistry, | ||
| // ready to be used to fetch secrets. | ||
| func NewProviderRegistryFromJSON(config ProviderRegistryConfig, jsonIn string) (*ProviderRegistry, error) { | ||
| var rawProviders []providerCandidate | ||
| err := json.Unmarshal([]byte(jsonIn), &rawProviders) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("unmarshalling secret providers: %w", err) | ||
| } | ||
|
|
||
| candidates := xsync.NewMapOf[providerCandidate]() | ||
| for _, provider := range rawProviders { | ||
| if _, ok := candidates.Load(provider.ID); ok { | ||
| return nil, fmt.Errorf("duplicate provider ID: %s. Provider IDs must be unique", provider.ID) | ||
| } | ||
|
|
||
| candidates.Store(provider.ID, provider) | ||
| } | ||
|
|
||
| return &ProviderRegistry{ | ||
| config: config, | ||
| candidates: candidates, | ||
| providers: xsync.NewMapOf[Provider](), | ||
| }, nil | ||
| } | ||
|
|
||
| func (pr *ProviderRegistry) FetchAll(configs []SecretConfig) ([]Secret, []error) { | ||
| secrets := make([]Secret, 0, len(configs)) | ||
| secretsCh := make(chan Secret) | ||
|
|
||
| errors := make([]error, 0, len(configs)) | ||
| errorsCh := make(chan error) | ||
|
|
||
| var wg sync.WaitGroup | ||
| for _, c := range configs { | ||
| wg.Add(1) | ||
| go func(config SecretConfig) { | ||
| defer wg.Done() | ||
|
|
||
| secret, err := pr.Fetch(config) | ||
| if err != nil { | ||
|
|
||
| errorsCh <- err | ||
| return | ||
| } | ||
| secretsCh <- secret | ||
| }(c) | ||
| } | ||
|
|
||
| go func() { | ||
| for err := range errorsCh { | ||
| errors = append(errors, err) | ||
| } | ||
| }() | ||
|
|
||
| go func() { | ||
| for secret := range secretsCh { | ||
| secrets = append(secrets, secret) | ||
| } | ||
| }() | ||
|
|
||
| wg.Wait() | ||
| close(secretsCh) | ||
| close(errorsCh) | ||
|
|
||
| return secrets, errors | ||
| } | ||
|
|
||
| // Fetch takes a SecretConfig, and attempts to fetch it from the provider specified in the config. | ||
| // This method is goroutine-safe. | ||
| func (pr *ProviderRegistry) Fetch(config SecretConfig) (Secret, error) { | ||
| if provider, ok := pr.providers.Load(config.ProviderID); ok { // We've used this provider before, it's already been initialized | ||
| value, err := provider.Fetch(config.Key) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("fetching secret %s from provider %s: %w", config.Key, config.ProviderID, err) | ||
| } | ||
|
|
||
| pr.config.Shell.Commentf("Secret %s fetched from provider %s", config.Key, config.ProviderID) | ||
| secret, err := newSecret(config, pr.config.Shell.Env, value) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("creating secret %s from provider %s: %w", config.Key, config.ProviderID, err) | ||
| } | ||
| return secret, nil | ||
| } | ||
|
|
||
| if candidate, ok := pr.candidates.Load(config.ProviderID); ok { // We haven't used this provider yet, so we need to initialize it | ||
| provider, err := candidate.Initialize() | ||
| if err != nil { | ||
| return nil, fmt.Errorf("initializing provider %s (type: %s) to fetch secret %s: %w", config.ProviderID, candidate.Type, config.Key, err) | ||
| } | ||
|
|
||
| pr.providers.Store(config.ProviderID, provider) // Store the initialised provider | ||
|
|
||
| value, err := provider.Fetch(config.Key) // Now fetch the actual secret. | ||
| if err != nil { | ||
| return nil, fmt.Errorf("fetching secret %s from provider %s: %w", config.Key, config.ProviderID, err) | ||
| } | ||
|
|
||
| pr.config.Shell.Commentf("Secret %s fetched from provider %s", config.Key, config.ProviderID) | ||
| secret, err := newSecret(config, pr.config.Shell.Env, value) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("creating secret %s from provider %s: %w", config.Key, config.ProviderID, err) | ||
| } | ||
| return secret, nil | ||
| } | ||
|
|
||
| // If we've got to this point, the user has tried to use a provider ID that's not in the registry, so we can't give them their secret | ||
| return nil, fmt.Errorf("no secret provider with ID: %s", config.ProviderID) | ||
| } |
Oops, something went wrong.