diff --git a/agent/agent_configuration.go b/agent/agent_configuration.go index 07d407c0ea..aae24019c8 100644 --- a/agent/agent_configuration.go +++ b/agent/agent_configuration.go @@ -4,7 +4,7 @@ package agent // has been loaded from the config file and command-line params type AgentConfiguration struct { ConfigPath string - BootstrapScript string + JobExecutorScript string BuildPath string HooksPath string GitMirrorsPath string diff --git a/agent/integration/job_runner_integration_test.go b/agent/integration/job_runner_integration_test.go index 951eb43e9c..689f881b76 100644 --- a/agent/integration/job_runner_integration_test.go +++ b/agent/integration/job_runner_integration_test.go @@ -41,7 +41,7 @@ func TestJobRunner_WhenJobHasToken_ItOverridesAccessToken(t *testing.T) { }) } -func TestJobRunnerPassesAccessTokenToBootstrap(t *testing.T) { +func TestJobRunnerPassesAccessTokenToJobExecute(t *testing.T) { ag := &api.AgentRegisterResponse{ AccessToken: "llamasrock", } @@ -91,20 +91,20 @@ func TestJobRunnerIgnoresPipelineChangesToProtectedVars(t *testing.T) { } -func runJob(t *testing.T, ag *api.AgentRegisterResponse, j *api.Job, cfg agent.AgentConfiguration, bootstrap func(c *bintest.Call)) { +func runJob(t *testing.T, ag *api.AgentRegisterResponse, j *api.Job, cfg agent.AgentConfiguration, executor func(c *bintest.Call)) { // create a mock agent API server := createTestAgentEndpoint(t, "my-job-id") defer server.Close() - // set up a mock bootstrap that the runner will call - bs, err := bintest.NewMock("buildkite-agent-bootstrap") + // set up a mock executor that the runner will call + bs, err := bintest.NewMock("buildkite-agent-job-execute") if err != nil { t.Fatalf("bintest.NewMock() error = %v", err) } defer bs.CheckAndClose(t) - // execute the callback we have inside the bootstrap mock - bs.Expect().Once().AndExitWith(0).AndCallFunc(bootstrap) + // execute the callback we have inside the executor mock + bs.Expect().Once().AndExitWith(0).AndCallFunc(executor) l := logger.Discard @@ -112,8 +112,8 @@ func runJob(t *testing.T, ag *api.AgentRegisterResponse, j *api.Job, cfg agent.A m := metrics.NewCollector(l, metrics.CollectorConfig{}) scope := m.Scope(metrics.Tags{}) - // set the bootstrap into the config - cfg.BootstrapScript = bs.Path + // set the executor into the config + cfg.JobExecutorScript = bs.Path client := api.NewClient(l, api.Config{ Endpoint: server.URL, diff --git a/agent/job_runner.go b/agent/job_runner.go index 43eedfce99..9055db40f5 100644 --- a/agent/job_runner.go +++ b/agent/job_runner.go @@ -24,7 +24,7 @@ import ( const ( // BuildkiteMessageMax is the maximum length of "BUILDKITE_MESSAGE=...\0" - // environment entry passed to bootstrap, beyond which it will be truncated + // environment entry passed to job executor, beyond which it will be truncated // to avoid exceeding the system limit. Note that it includes the variable // name, equals sign, and null terminator. // @@ -151,11 +151,11 @@ func NewJobRunner(l logger.Logger, scope *metrics.Scope, ag *api.AgentRegisterRe return nil, err } - // The bootstrap-script gets parsed based on the operating system - cmd, err := shellwords.Split(conf.AgentConfiguration.BootstrapScript) + // The job executor script gets parsed based on the operating system + cmd, err := shellwords.Split(conf.AgentConfiguration.JobExecutorScript) if err != nil { - return nil, fmt.Errorf("Failed to split bootstrap-script (%q) into tokens: %v", - conf.AgentConfiguration.BootstrapScript, err) + return nil, fmt.Errorf("Failed to split job executor script (%q) into tokens: %v", + conf.AgentConfiguration.JobExecutorScript, err) } // Our log streamer works off a buffer of output @@ -237,7 +237,7 @@ func NewJobRunner(l logger.Logger, scope *metrics.Scope, ag *api.AgentRegisterRe // take precedence over the agent processEnv := append(os.Environ(), env...) - // The process that will run the bootstrap script + // The process that will run the job executor script runner.process = process.New(l, process.Config{ Path: cmd[0], Args: cmd[1:], diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go index 9c96d244a1..6832f3f4af 100644 --- a/clicommand/agent_start.go +++ b/clicommand/agent_start.go @@ -40,7 +40,7 @@ const startDescription = `Usage: Description: - When a job is ready to run it will call the "bootstrap-script" + When a job is ready to run it will call the "job-executor-script" and pass it all the environment variables required for the job to run. This script is responsible for checking out the code, and running the actual build script defined in the pipeline. @@ -54,8 +54,8 @@ Example: // Adding config requires changes in a few different spots // - The AgentStartConfig struct with a cli parameter // - As a flag in the AgentStartCommand (with matching env) -// - Into an env to be passed to the bootstrap in agent/job_runner.go, createEnvironment() -// - Into clicommand/bootstrap.go to read it from the env into the bootstrap config +// - Into an env to be passed to the job executor in agent/job_runner.go, createEnvironment() +// - Into clicommand/exec-job.go to read it from the env into the job executor config type AgentStartConfig struct { Config string `cli:"config"` @@ -64,7 +64,7 @@ type AgentStartConfig struct { AcquireJob string `cli:"acquire-job"` DisconnectAfterJob bool `cli:"disconnect-after-job"` DisconnectAfterIdleTimeout int `cli:"disconnect-after-idle-timeout"` - BootstrapScript string `cli:"bootstrap-script" normalize:"commandpath"` + JobExecutorScript string `cli:"job-executor-script" normalize:"commandpath"` CancelGracePeriod int `cli:"cancel-grace-period"` EnableJobLogTmpfile bool `cli:"enable-job-log-tmpfile"` BuildPath string `cli:"build-path" normalize:"filepath" validate:"required"` @@ -134,6 +134,7 @@ type AgentStartConfig struct { MetaDataGCP bool `cli:"meta-data-gcp" deprecated-and-renamed-to:"TagsFromGCP"` TagsFromEC2 bool `cli:"tags-from-ec2" deprecated-and-renamed-to:"TagsFromEC2MetaData"` TagsFromGCP bool `cli:"tags-from-gcp" deprecated-and-renamed-to:"TagsFromGCPMetaData"` + BootstrapScript string `cli:"bootstrap-script" deprecated-and-renamed-to:"JobExecutorScript" normalize:"commandpath"` DisconnectAfterJobTimeout int `cli:"disconnect-after-job-timeout" deprecated:"Use disconnect-after-idle-timeout instead"` } @@ -417,9 +418,15 @@ var AgentStartCommand = cli.Command{ cli.StringFlag{ Name: "bootstrap-script", Value: "", - Usage: "The command that is executed for bootstrapping a job, defaults to the bootstrap sub-command of this binary", + Usage: "The command that is executed for bootstrapping a job, defaults to the exec-job sub-command of this binary", EnvVar: "BUILDKITE_BOOTSTRAP_SCRIPT_PATH", }, + cli.StringFlag{ + Name: "job-executor-script", + Value: "", + Usage: "The command that is executed for running a job, defaults to the exec-job sub-command of this binary", + EnvVar: "BUILDKITE_JOB_EXECUTOR_SCRIPT_PATH", + }, cli.StringFlag{ Name: "build-path", Value: "", @@ -630,7 +637,7 @@ var AgentStartCommand = cli.Command{ done := HandleGlobalFlags(l, cfg) defer done() - // Remove any config env from the environment to prevent them propagating to bootstrap + // Remove any config env from the environment to prevent them propagating to job execution err = UnsetConfigFromEnvironment(c) if err != nil { fmt.Printf("%s", err) @@ -650,12 +657,12 @@ var AgentStartCommand = cli.Command{ } // Set a useful default for the bootstrap script - if cfg.BootstrapScript == "" { + if cfg.JobExecutorScript == "" { exePath, err := os.Executable() if err != nil { - l.Fatal("Unable to find executable path for bootstrap") + l.Fatal("Unable to find our executable path to construct the job executor script: %v", err) } - cfg.BootstrapScript = fmt.Sprintf("%s bootstrap", shellwords.Quote(exePath)) + cfg.JobExecutorScript = fmt.Sprintf("%s exec-job", shellwords.Quote(exePath)) } isSetNoPlugins := c.IsSet("no-plugins") @@ -738,14 +745,14 @@ var AgentStartCommand = cli.Command{ DatadogDistributions: cfg.MetricsDatadogDistributions, }) - // Sense check supported tracing backends, we don't want bootstrapped jobs to silently have no tracing + // Sense check supported tracing backends, we don't want jobs to silently have no tracing if _, has := tracetools.ValidTracingBackends[cfg.TracingBackend]; !has { l.Fatal("The given tracing backend %q is not supported. Valid backends are: %q", cfg.TracingBackend, maps.Keys(tracetools.ValidTracingBackends)) } // AgentConfiguration is the runtime configuration for an agent agentConf := agent.AgentConfiguration{ - BootstrapScript: cfg.BootstrapScript, + JobExecutorScript: cfg.JobExecutorScript, BuildPath: cfg.BuildPath, GitMirrorsPath: cfg.GitMirrorsPath, GitMirrorsLockTimeout: cfg.GitMirrorsLockTimeout, @@ -807,7 +814,7 @@ var AgentStartCommand = cli.Command{ l.WithFields(logger.StringField(`path`, agentConf.ConfigPath)).Info("Configuration loaded") } - l.Debug("Bootstrap command: %s", agentConf.BootstrapScript) + l.Debug("Job Exec command: %s", agentConf.JobExecutorScript) l.Debug("Build path: %s", agentConf.BuildPath) l.Debug("Hooks directory: %s", agentConf.HooksPath) l.Debug("Plugins directory: %s", agentConf.PluginsPath) @@ -841,7 +848,7 @@ var AgentStartCommand = cli.Command{ l.Fatal("Failed to parse cancel-signal: %v", err) } - // confirm the BuildPath is exists. The bootstrap is going to write to it when a job executes, + // confirm the BuildPath is exists. The job executor is going to write to it when a job executes, // so we may as well check that'll work now and fail early if it's a problem if !utils.FileExists(agentConf.BuildPath) { l.Info("Build Path doesn't exist, creating it (%s)", agentConf.BuildPath) diff --git a/clicommand/bootstrap.go b/clicommand/bootstrap.go deleted file mode 100644 index 8d36882dfd..0000000000 --- a/clicommand/bootstrap.go +++ /dev/null @@ -1,512 +0,0 @@ -package clicommand - -import ( - "context" - "fmt" - "os" - "os/signal" - "runtime" - "sync" - "syscall" - - "github.com/buildkite/agent/v3/cliconfig" - "github.com/buildkite/agent/v3/experiments" - "github.com/buildkite/agent/v3/job" - "github.com/buildkite/agent/v3/process" - "github.com/urfave/cli" -) - -const bootstrapHelpDescription = `Usage: - - buildkite-agent bootstrap [options...] - -Description: - - The bootstrap command executes a Buildkite job locally. - - Generally the bootstrap command is run as a sub-process of the buildkite-agent to execute - a given job sent from buildkite.com, but you can also invoke the bootstrap manually. - - Execution is broken down into phases. By default, the bootstrap runs a plugin phase which - sets up any plugins specified, then a checkout phase which pulls down your code and then a - command phase that executes the specified command in the created environment. - - You can run only specific phases with the --phases flag. - - The bootstrap is also responsible for executing hooks around the phases. - See https://buildkite.com/docs/agent/v3/hooks for more details. - -Example: - - $ eval $(curl -s -H "Authorization: Bearer xxx" \ - "https://api.buildkite.com/v2/organizations/[org]/pipelines/[proj]/builds/[build]/jobs/[job]/env.txt" | sed 's/^/export /') - $ buildkite-agent bootstrap --build-path builds` - -type BootstrapConfig struct { - Command string `cli:"command"` - JobID string `cli:"job" validate:"required"` - Repository string `cli:"repository" validate:"required"` - Commit string `cli:"commit" validate:"required"` - Branch string `cli:"branch" validate:"required"` - Tag string `cli:"tag"` - RefSpec string `cli:"refspec"` - Plugins string `cli:"plugins"` - PullRequest string `cli:"pullrequest"` - GitSubmodules bool `cli:"git-submodules"` - SSHKeyscan bool `cli:"ssh-keyscan"` - AgentName string `cli:"agent" validate:"required"` - Queue string `cli:"queue"` - OrganizationSlug string `cli:"organization" validate:"required"` - PipelineSlug string `cli:"pipeline" validate:"required"` - PipelineProvider string `cli:"pipeline-provider" validate:"required"` - AutomaticArtifactUploadPaths string `cli:"artifact-upload-paths"` - ArtifactUploadDestination string `cli:"artifact-upload-destination"` - CleanCheckout bool `cli:"clean-checkout"` - GitCheckoutFlags string `cli:"git-checkout-flags"` - GitCloneFlags string `cli:"git-clone-flags"` - GitFetchFlags string `cli:"git-fetch-flags"` - GitCloneMirrorFlags string `cli:"git-clone-mirror-flags"` - GitCleanFlags string `cli:"git-clean-flags"` - GitMirrorsPath string `cli:"git-mirrors-path" normalize:"filepath"` - GitMirrorsLockTimeout int `cli:"git-mirrors-lock-timeout"` - GitMirrorsSkipUpdate bool `cli:"git-mirrors-skip-update"` - GitSubmoduleCloneConfig []string `cli:"git-submodule-clone-config"` - BinPath string `cli:"bin-path" normalize:"filepath"` - BuildPath string `cli:"build-path" normalize:"filepath"` - HooksPath string `cli:"hooks-path" normalize:"filepath"` - PluginsPath string `cli:"plugins-path" normalize:"filepath"` - CommandEval bool `cli:"command-eval"` - PluginsEnabled bool `cli:"plugins-enabled"` - PluginValidation bool `cli:"plugin-validation"` - PluginsAlwaysCloneFresh bool `cli:"plugins-always-clone-fresh"` - LocalHooksEnabled bool `cli:"local-hooks-enabled"` - PTY bool `cli:"pty"` - LogLevel string `cli:"log-level"` - Debug bool `cli:"debug"` - Shell string `cli:"shell"` - Experiments []string `cli:"experiment" normalize:"list"` - Phases []string `cli:"phases" normalize:"list"` - Profile string `cli:"profile"` - CancelSignal string `cli:"cancel-signal"` - RedactedVars []string `cli:"redacted-vars" normalize:"list"` - TracingBackend string `cli:"tracing-backend"` - TracingServiceName string `cli:"tracing-service-name"` -} - -var BootstrapCommand = cli.Command{ - Name: "bootstrap", - Usage: "Run a Buildkite job locally", - Description: bootstrapHelpDescription, - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "command", - Value: "", - Usage: "The command to run", - EnvVar: "BUILDKITE_COMMAND", - }, - cli.StringFlag{ - Name: "job", - Value: "", - Usage: "The ID of the job being run", - EnvVar: "BUILDKITE_JOB_ID", - }, - cli.StringFlag{ - Name: "repository", - Value: "", - Usage: "The repository to clone and run the job from", - EnvVar: "BUILDKITE_REPO", - }, - cli.StringFlag{ - Name: "commit", - Value: "", - Usage: "The commit to checkout in the repository", - EnvVar: "BUILDKITE_COMMIT", - }, - cli.StringFlag{ - Name: "branch", - Value: "", - Usage: "The branch the commit is in", - EnvVar: "BUILDKITE_BRANCH", - }, - cli.StringFlag{ - Name: "tag", - Value: "", - Usage: "The tag the commit", - EnvVar: "BUILDKITE_TAG", - }, - cli.StringFlag{ - Name: "refspec", - Value: "", - Usage: "Optional refspec to override git fetch", - EnvVar: "BUILDKITE_REFSPEC", - }, - cli.StringFlag{ - Name: "plugins", - Value: "", - Usage: "The plugins for the job", - EnvVar: "BUILDKITE_PLUGINS", - }, - cli.StringFlag{ - Name: "pullrequest", - Value: "", - Usage: "The number/id of the pull request this commit belonged to", - EnvVar: "BUILDKITE_PULL_REQUEST", - }, - cli.StringFlag{ - Name: "agent", - Value: "", - Usage: "The name of the agent running the job", - EnvVar: "BUILDKITE_AGENT_NAME", - }, - cli.StringFlag{ - Name: "queue", - Value: "", - Usage: "The name of the queue the agent belongs to, if tagged", - EnvVar: "BUILDKITE_AGENT_META_DATA_QUEUE", - }, - cli.StringFlag{ - Name: "organization", - Value: "", - Usage: "The slug of the organization that the job is a part of", - EnvVar: "BUILDKITE_ORGANIZATION_SLUG", - }, - cli.StringFlag{ - Name: "pipeline", - Value: "", - Usage: "The slug of the pipeline that the job is a part of", - EnvVar: "BUILDKITE_PIPELINE_SLUG", - }, - cli.StringFlag{ - Name: "pipeline-provider", - Value: "", - Usage: "The id of the SCM provider that the repository is hosted on", - EnvVar: "BUILDKITE_PIPELINE_PROVIDER", - }, - cli.StringFlag{ - Name: "artifact-upload-paths", - Value: "", - Usage: "Paths to files to automatically upload at the end of a job", - EnvVar: "BUILDKITE_ARTIFACT_PATHS", - }, - cli.StringFlag{ - Name: "artifact-upload-destination", - Value: "", - Usage: "A custom location to upload artifact paths to (for example, s3://my-custom-bucket/and/prefix)", - EnvVar: "BUILDKITE_ARTIFACT_UPLOAD_DESTINATION", - }, - cli.BoolFlag{ - Name: "clean-checkout", - Usage: "Whether or not the bootstrap should remove the existing repository before running the command", - EnvVar: "BUILDKITE_CLEAN_CHECKOUT", - }, - cli.StringFlag{ - Name: "git-checkout-flags", - Value: "-f", - Usage: "Flags to pass to \"git checkout\" command", - EnvVar: "BUILDKITE_GIT_CHECKOUT_FLAGS", - }, - cli.StringFlag{ - Name: "git-clone-flags", - Value: "-v", - Usage: "Flags to pass to \"git clone\" command", - EnvVar: "BUILDKITE_GIT_CLONE_FLAGS", - }, - cli.StringFlag{ - Name: "git-clone-mirror-flags", - Value: "-v", - Usage: "Flags to pass to \"git clone\" command when mirroring", - EnvVar: "BUILDKITE_GIT_CLONE_MIRROR_FLAGS", - }, - cli.StringFlag{ - Name: "git-clean-flags", - Value: "-ffxdq", - Usage: "Flags to pass to \"git clean\" command", - EnvVar: "BUILDKITE_GIT_CLEAN_FLAGS", - }, - cli.StringFlag{ - Name: "git-fetch-flags", - Value: "", - Usage: "Flags to pass to \"git fetch\" command", - EnvVar: "BUILDKITE_GIT_FETCH_FLAGS", - }, - cli.StringSliceFlag{ - Name: "git-submodule-clone-config", - Value: &cli.StringSlice{}, - Usage: "Comma separated key=value git config pairs applied before git submodule clone commands, e.g. `update --init`. If the config is needed to be applied to all git commands, supply it in a global git config file for the system that the agent runs in instead.", - EnvVar: "BUILDKITE_GIT_SUBMODULE_CLONE_CONFIG", - }, - cli.StringFlag{ - Name: "git-mirrors-path", - Value: "", - Usage: "Path to where mirrors of git repositories are stored", - EnvVar: "BUILDKITE_GIT_MIRRORS_PATH", - }, - cli.IntFlag{ - Name: "git-mirrors-lock-timeout", - Value: 300, - Usage: "Seconds to lock a git mirror during clone, should exceed your longest checkout", - EnvVar: "BUILDKITE_GIT_MIRRORS_LOCK_TIMEOUT", - }, - cli.BoolFlag{ - Name: "git-mirrors-skip-update", - Usage: "Skip updating the Git mirror", - EnvVar: "BUILDKITE_GIT_MIRRORS_SKIP_UPDATE", - }, - cli.StringFlag{ - Name: "bin-path", - Value: "", - Usage: "Directory where the buildkite-agent binary lives", - EnvVar: "BUILDKITE_BIN_PATH", - }, - cli.StringFlag{ - Name: "build-path", - Value: "", - Usage: "Directory where builds will be created", - EnvVar: "BUILDKITE_BUILD_PATH", - }, - cli.StringFlag{ - Name: "hooks-path", - Value: "", - Usage: "Directory where the hook scripts are found", - EnvVar: "BUILDKITE_HOOKS_PATH", - }, - cli.StringFlag{ - Name: "plugins-path", - Value: "", - Usage: "Directory where the plugins are saved to", - EnvVar: "BUILDKITE_PLUGINS_PATH", - }, - cli.BoolTFlag{ - Name: "command-eval", - Usage: "Allow running of arbitrary commands", - EnvVar: "BUILDKITE_COMMAND_EVAL", - }, - cli.BoolTFlag{ - Name: "plugins-enabled", - Usage: "Allow plugins to be run", - EnvVar: "BUILDKITE_PLUGINS_ENABLED", - }, - cli.BoolFlag{ - Name: "plugin-validation", - Usage: "Validate plugin configuration", - EnvVar: "BUILDKITE_PLUGIN_VALIDATION", - }, - cli.BoolFlag{ - Name: "plugins-always-clone-fresh", - Usage: "Always make a new clone of plugin source, even if already present", - EnvVar: "BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH", - }, - cli.BoolTFlag{ - Name: "local-hooks-enabled", - Usage: "Allow local hooks to be run", - EnvVar: "BUILDKITE_LOCAL_HOOKS_ENABLED", - }, - cli.BoolTFlag{ - Name: "ssh-keyscan", - Usage: "Automatically run ssh-keyscan before checkout", - EnvVar: "BUILDKITE_SSH_KEYSCAN", - }, - cli.BoolTFlag{ - Name: "git-submodules", - Usage: "Enable git submodules", - EnvVar: "BUILDKITE_GIT_SUBMODULES", - }, - cli.BoolTFlag{ - Name: "pty", - Usage: "Run jobs within a pseudo terminal", - EnvVar: "BUILDKITE_PTY", - }, - cli.StringFlag{ - Name: "shell", - Usage: "The shell to use to interpret build commands", - EnvVar: "BUILDKITE_SHELL", - Value: DefaultShell(), - }, - cli.StringSliceFlag{ - Name: "phases", - Usage: "The specific phases to execute. The order they're defined is irrelevant.", - EnvVar: "BUILDKITE_BOOTSTRAP_PHASES", - }, - cli.StringFlag{ - Name: "cancel-signal", - Usage: "The signal to use for cancellation", - EnvVar: "BUILDKITE_CANCEL_SIGNAL", - Value: "SIGTERM", - }, - cli.StringSliceFlag{ - Name: "redacted-vars", - Usage: "Pattern of environment variable names containing sensitive values", - EnvVar: "BUILDKITE_REDACTED_VARS", - }, - cli.StringFlag{ - Name: "tracing-backend", - Usage: "The name of the tracing backend to use.", - EnvVar: "BUILDKITE_TRACING_BACKEND", - Value: "", - }, - cli.StringFlag{ - Name: "tracing-service-name", - Usage: "Service name to use when reporting traces.", - EnvVar: "BUILDKITE_TRACING_SERVICE_NAME", - Value: "buildkite-agent", - }, - DebugFlag, - LogLevelFlag, - ExperimentsFlag, - ProfileFlag, - }, - Action: func(c *cli.Context) { - // The configuration will be loaded into this struct - cfg := BootstrapConfig{} - - loader := cliconfig.Loader{CLI: c, Config: &cfg} - warnings, err := loader.Load() - if err != nil { - fmt.Printf("%s", err) - os.Exit(1) - } - - l := CreateLogger(&cfg) - - // Now that we have a logger, log out the warnings that loading config generated - for _, warning := range warnings { - l.Warn("%s", warning) - } - - // Enable experiments - for _, name := range cfg.Experiments { - experiments.Enable(name) - } - - // Handle profiling flag - done := HandleProfileFlag(l, cfg) - defer done() - - // Turn of PTY support if we're on Windows - runInPty := cfg.PTY - if runtime.GOOS == "windows" { - runInPty = false - } - - // Validate phases - for _, phase := range cfg.Phases { - switch phase { - case "plugin", "checkout", "command": - // Valid phase - default: - l.Fatal("Invalid phase %q", phase) - } - } - - cancelSig, err := process.ParseSignal(cfg.CancelSignal) - if err != nil { - l.Fatal("Failed to parse cancel-signal: %v", err) - } - - // Configure the bootstraper - bootstrap := job.New(job.Config{ - AgentName: cfg.AgentName, - ArtifactUploadDestination: cfg.ArtifactUploadDestination, - AutomaticArtifactUploadPaths: cfg.AutomaticArtifactUploadPaths, - BinPath: cfg.BinPath, - Branch: cfg.Branch, - BuildPath: cfg.BuildPath, - CancelSignal: cancelSig, - CleanCheckout: cfg.CleanCheckout, - Command: cfg.Command, - CommandEval: cfg.CommandEval, - Commit: cfg.Commit, - Debug: cfg.Debug, - GitCheckoutFlags: cfg.GitCheckoutFlags, - GitCleanFlags: cfg.GitCleanFlags, - GitCloneFlags: cfg.GitCloneFlags, - GitCloneMirrorFlags: cfg.GitCloneMirrorFlags, - GitFetchFlags: cfg.GitFetchFlags, - GitMirrorsLockTimeout: cfg.GitMirrorsLockTimeout, - GitMirrorsPath: cfg.GitMirrorsPath, - GitMirrorsSkipUpdate: cfg.GitMirrorsSkipUpdate, - GitSubmodules: cfg.GitSubmodules, - GitSubmoduleCloneConfig: cfg.GitSubmoduleCloneConfig, - HooksPath: cfg.HooksPath, - JobID: cfg.JobID, - LocalHooksEnabled: cfg.LocalHooksEnabled, - OrganizationSlug: cfg.OrganizationSlug, - Phases: cfg.Phases, - PipelineProvider: cfg.PipelineProvider, - PipelineSlug: cfg.PipelineSlug, - PluginValidation: cfg.PluginValidation, - Plugins: cfg.Plugins, - PluginsEnabled: cfg.PluginsEnabled, - PluginsAlwaysCloneFresh: cfg.PluginsAlwaysCloneFresh, - PluginsPath: cfg.PluginsPath, - PullRequest: cfg.PullRequest, - Queue: cfg.Queue, - RedactedVars: cfg.RedactedVars, - RefSpec: cfg.RefSpec, - Repository: cfg.Repository, - RunInPty: runInPty, - SSHKeyscan: cfg.SSHKeyscan, - Shell: cfg.Shell, - Tag: cfg.Tag, - TracingBackend: cfg.TracingBackend, - TracingServiceName: cfg.TracingServiceName, - }) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - signals := make(chan os.Signal, 1) - signal.Notify(signals, os.Interrupt, - syscall.SIGHUP, - syscall.SIGTERM, - syscall.SIGINT, - syscall.SIGQUIT) - defer signal.Stop(signals) - - var ( - cancelled bool - received os.Signal - signalMu sync.Mutex - ) - - // Listen for signals in the background and cancel the bootstrap - go func() { - sig := <-signals - signalMu.Lock() - defer signalMu.Unlock() - - // Cancel the bootstrap - bootstrap.Cancel() - - // Track the state and signal used - cancelled = true - received = sig - - // Remove our signal handler so subsequent signals kill - signal.Stop(signals) - }() - - // Run the bootstrap and get the exit code - exitCode := bootstrap.Run(ctx) - - signalMu.Lock() - defer signalMu.Unlock() - - // If cancelled and our child process returns a non-zero, we should terminate - // ourselves with the same signal so that our caller can detect and handle appropriately - if cancelled && runtime.GOOS != "windows" { - p, err := os.FindProcess(os.Getpid()) - if err != nil { - l.Error("Failed to find current process: %v", err) - } - - l.Debug("Terminating bootstrap after cancellation with %v", received) - err = p.Signal(received) - if err != nil { - l.Error("Failed to signal self: %v", err) - } - } - - os.Exit(exitCode) - }, -} diff --git a/clicommand/exec-job.go b/clicommand/exec-job.go new file mode 100644 index 0000000000..2b667ba46a --- /dev/null +++ b/clicommand/exec-job.go @@ -0,0 +1,562 @@ +package clicommand + +import ( + "context" + "fmt" + "os" + "os/signal" + "runtime" + "strings" + "sync" + "syscall" + "text/template" + + "github.com/buildkite/agent/v3/cliconfig" + "github.com/buildkite/agent/v3/experiments" + "github.com/buildkite/agent/v3/job" + "github.com/buildkite/agent/v3/process" + "github.com/urfave/cli" +) + +var execJobHelpTpl = template.Must(template.New("execJobHelp").Parse(`Usage: + + buildkite-agent {{.}} [options...] + +Description: + + The {{.}} command executes a Buildkite job locally. + + Generally the {{.}} command is run as a sub-process of the buildkite-agent to execute + a given job sent from buildkite.com, but you can also invoke the {{.}} manually. + + Execution is broken down into phases. By default, the {{.}} runs a plugin phase which + sets up any plugins specified, then a checkout phase which pulls down your code and then a + command phase that executes the specified command in the created environment. + + You can run only specific phases with the --phases flag. + + The {{.}} is also responsible for executing hooks around the phases. + See https://buildkite.com/docs/agent/v3/hooks for more details. + +Example: + + $ eval $(curl -s -H "Authorization: Bearer xxx" \ + "https://api.buildkite.com/v2/organizations/[org]/pipelines/[proj]/builds/[build]/jobs/[job]/env.txt" | sed 's/^/export /') + $ buildkite-agent {{.}} --build-path builds`)) + +type ExecJobConfig struct { + Command string `cli:"command"` + JobID string `cli:"job" validate:"required"` + Repository string `cli:"repository" validate:"required"` + Commit string `cli:"commit" validate:"required"` + Branch string `cli:"branch" validate:"required"` + Tag string `cli:"tag"` + RefSpec string `cli:"refspec"` + Plugins string `cli:"plugins"` + PullRequest string `cli:"pullrequest"` + GitSubmodules bool `cli:"git-submodules"` + SSHKeyscan bool `cli:"ssh-keyscan"` + AgentName string `cli:"agent" validate:"required"` + Queue string `cli:"queue"` + OrganizationSlug string `cli:"organization" validate:"required"` + PipelineSlug string `cli:"pipeline" validate:"required"` + PipelineProvider string `cli:"pipeline-provider" validate:"required"` + AutomaticArtifactUploadPaths string `cli:"artifact-upload-paths"` + ArtifactUploadDestination string `cli:"artifact-upload-destination"` + CleanCheckout bool `cli:"clean-checkout"` + GitCheckoutFlags string `cli:"git-checkout-flags"` + GitCloneFlags string `cli:"git-clone-flags"` + GitFetchFlags string `cli:"git-fetch-flags"` + GitCloneMirrorFlags string `cli:"git-clone-mirror-flags"` + GitCleanFlags string `cli:"git-clean-flags"` + GitMirrorsPath string `cli:"git-mirrors-path" normalize:"filepath"` + GitMirrorsLockTimeout int `cli:"git-mirrors-lock-timeout"` + GitMirrorsSkipUpdate bool `cli:"git-mirrors-skip-update"` + GitSubmoduleCloneConfig []string `cli:"git-submodule-clone-config"` + BinPath string `cli:"bin-path" normalize:"filepath"` + BuildPath string `cli:"build-path" normalize:"filepath"` + HooksPath string `cli:"hooks-path" normalize:"filepath"` + PluginsPath string `cli:"plugins-path" normalize:"filepath"` + CommandEval bool `cli:"command-eval"` + PluginsEnabled bool `cli:"plugins-enabled"` + PluginValidation bool `cli:"plugin-validation"` + PluginsAlwaysCloneFresh bool `cli:"plugins-always-clone-fresh"` + LocalHooksEnabled bool `cli:"local-hooks-enabled"` + PTY bool `cli:"pty"` + LogLevel string `cli:"log-level"` + Debug bool `cli:"debug"` + Shell string `cli:"shell"` + Experiments []string `cli:"experiment" normalize:"list"` + Phases []string `cli:"phases" normalize:"list"` + Profile string `cli:"profile"` + CancelSignal string `cli:"cancel-signal"` + RedactedVars []string `cli:"redacted-vars" normalize:"list"` + TracingBackend string `cli:"tracing-backend"` + TracingServiceName string `cli:"tracing-service-name"` +} + +var execJobFlags = []cli.Flag{ + cli.StringFlag{ + Name: "command", + Value: "", + Usage: "The command to run", + EnvVar: "BUILDKITE_COMMAND", + }, + cli.StringFlag{ + Name: "job", + Value: "", + Usage: "The ID of the job being run", + EnvVar: "BUILDKITE_JOB_ID", + }, + cli.StringFlag{ + Name: "repository", + Value: "", + Usage: "The repository to clone and run the job from", + EnvVar: "BUILDKITE_REPO", + }, + cli.StringFlag{ + Name: "commit", + Value: "", + Usage: "The commit to checkout in the repository", + EnvVar: "BUILDKITE_COMMIT", + }, + cli.StringFlag{ + Name: "branch", + Value: "", + Usage: "The branch the commit is in", + EnvVar: "BUILDKITE_BRANCH", + }, + cli.StringFlag{ + Name: "tag", + Value: "", + Usage: "The tag the commit", + EnvVar: "BUILDKITE_TAG", + }, + cli.StringFlag{ + Name: "refspec", + Value: "", + Usage: "Optional refspec to override git fetch", + EnvVar: "BUILDKITE_REFSPEC", + }, + cli.StringFlag{ + Name: "plugins", + Value: "", + Usage: "The plugins for the job", + EnvVar: "BUILDKITE_PLUGINS", + }, + cli.StringFlag{ + Name: "pullrequest", + Value: "", + Usage: "The number/id of the pull request this commit belonged to", + EnvVar: "BUILDKITE_PULL_REQUEST", + }, + cli.StringFlag{ + Name: "agent", + Value: "", + Usage: "The name of the agent running the job", + EnvVar: "BUILDKITE_AGENT_NAME", + }, + cli.StringFlag{ + Name: "queue", + Value: "", + Usage: "The name of the queue the agent belongs to, if tagged", + EnvVar: "BUILDKITE_AGENT_META_DATA_QUEUE", + }, + cli.StringFlag{ + Name: "organization", + Value: "", + Usage: "The slug of the organization that the job is a part of", + EnvVar: "BUILDKITE_ORGANIZATION_SLUG", + }, + cli.StringFlag{ + Name: "pipeline", + Value: "", + Usage: "The slug of the pipeline that the job is a part of", + EnvVar: "BUILDKITE_PIPELINE_SLUG", + }, + cli.StringFlag{ + Name: "pipeline-provider", + Value: "", + Usage: "The id of the SCM provider that the repository is hosted on", + EnvVar: "BUILDKITE_PIPELINE_PROVIDER", + }, + cli.StringFlag{ + Name: "artifact-upload-paths", + Value: "", + Usage: "Paths to files to automatically upload at the end of a job", + EnvVar: "BUILDKITE_ARTIFACT_PATHS", + }, + cli.StringFlag{ + Name: "artifact-upload-destination", + Value: "", + Usage: "A custom location to upload artifact paths to (for example, s3://my-custom-bucket/and/prefix)", + EnvVar: "BUILDKITE_ARTIFACT_UPLOAD_DESTINATION", + }, + cli.BoolFlag{ + Name: "clean-checkout", + Usage: "Whether or not the job executor should remove the existing repository before running the command", + EnvVar: "BUILDKITE_CLEAN_CHECKOUT", + }, + cli.StringFlag{ + Name: "git-checkout-flags", + Value: "-f", + Usage: "Flags to pass to \"git checkout\" command", + EnvVar: "BUILDKITE_GIT_CHECKOUT_FLAGS", + }, + cli.StringFlag{ + Name: "git-clone-flags", + Value: "-v", + Usage: "Flags to pass to \"git clone\" command", + EnvVar: "BUILDKITE_GIT_CLONE_FLAGS", + }, + cli.StringFlag{ + Name: "git-clone-mirror-flags", + Value: "-v", + Usage: "Flags to pass to \"git clone\" command when mirroring", + EnvVar: "BUILDKITE_GIT_CLONE_MIRROR_FLAGS", + }, + cli.StringFlag{ + Name: "git-clean-flags", + Value: "-ffxdq", + Usage: "Flags to pass to \"git clean\" command", + EnvVar: "BUILDKITE_GIT_CLEAN_FLAGS", + }, + cli.StringFlag{ + Name: "git-fetch-flags", + Value: "", + Usage: "Flags to pass to \"git fetch\" command", + EnvVar: "BUILDKITE_GIT_FETCH_FLAGS", + }, + cli.StringSliceFlag{ + Name: "git-submodule-clone-config", + Value: &cli.StringSlice{}, + Usage: "Comma separated key=value git config pairs applied before git submodule clone commands, e.g. `update --init`. If the config is needed to be applied to all git commands, supply it in a global git config file for the system that the agent runs in instead.", + EnvVar: "BUILDKITE_GIT_SUBMODULE_CLONE_CONFIG", + }, + cli.StringFlag{ + Name: "git-mirrors-path", + Value: "", + Usage: "Path to where mirrors of git repositories are stored", + EnvVar: "BUILDKITE_GIT_MIRRORS_PATH", + }, + cli.IntFlag{ + Name: "git-mirrors-lock-timeout", + Value: 300, + Usage: "Seconds to lock a git mirror during clone, should exceed your longest checkout", + EnvVar: "BUILDKITE_GIT_MIRRORS_LOCK_TIMEOUT", + }, + cli.BoolFlag{ + Name: "git-mirrors-skip-update", + Usage: "Skip updating the Git mirror", + EnvVar: "BUILDKITE_GIT_MIRRORS_SKIP_UPDATE", + }, + cli.StringFlag{ + Name: "bin-path", + Value: "", + Usage: "Directory where the buildkite-agent binary lives", + EnvVar: "BUILDKITE_BIN_PATH", + }, + cli.StringFlag{ + Name: "build-path", + Value: "", + Usage: "Directory where builds will be created", + EnvVar: "BUILDKITE_BUILD_PATH", + }, + cli.StringFlag{ + Name: "hooks-path", + Value: "", + Usage: "Directory where the hook scripts are found", + EnvVar: "BUILDKITE_HOOKS_PATH", + }, + cli.StringFlag{ + Name: "plugins-path", + Value: "", + Usage: "Directory where the plugins are saved to", + EnvVar: "BUILDKITE_PLUGINS_PATH", + }, + cli.BoolTFlag{ + Name: "command-eval", + Usage: "Allow running of arbitrary commands", + EnvVar: "BUILDKITE_COMMAND_EVAL", + }, + cli.BoolTFlag{ + Name: "plugins-enabled", + Usage: "Allow plugins to be run", + EnvVar: "BUILDKITE_PLUGINS_ENABLED", + }, + cli.BoolFlag{ + Name: "plugin-validation", + Usage: "Validate plugin configuration", + EnvVar: "BUILDKITE_PLUGIN_VALIDATION", + }, + cli.BoolFlag{ + Name: "plugins-always-clone-fresh", + Usage: "Always make a new clone of plugin source, even if already present", + EnvVar: "BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH", + }, + cli.BoolTFlag{ + Name: "local-hooks-enabled", + Usage: "Allow local hooks to be run", + EnvVar: "BUILDKITE_LOCAL_HOOKS_ENABLED", + }, + cli.BoolTFlag{ + Name: "ssh-keyscan", + Usage: "Automatically run ssh-keyscan before checkout", + EnvVar: "BUILDKITE_SSH_KEYSCAN", + }, + cli.BoolTFlag{ + Name: "git-submodules", + Usage: "Enable git submodules", + EnvVar: "BUILDKITE_GIT_SUBMODULES", + }, + cli.BoolTFlag{ + Name: "pty", + Usage: "Run jobs within a pseudo terminal", + EnvVar: "BUILDKITE_PTY", + }, + cli.StringFlag{ + Name: "shell", + Usage: "The shell to use to interpret build commands", + EnvVar: "BUILDKITE_SHELL", + Value: DefaultShell(), + }, + cli.StringSliceFlag{ + Name: "phases", + Usage: "The specific phases to execute. The order they're defined is irrelevant.", + EnvVar: "BUILDKITE_BOOTSTRAP_PHASES", + }, + cli.StringFlag{ + Name: "cancel-signal", + Usage: "The signal to use for cancellation", + EnvVar: "BUILDKITE_CANCEL_SIGNAL", + Value: "SIGTERM", + }, + cli.StringSliceFlag{ + Name: "redacted-vars", + Usage: "Pattern of environment variable names containing sensitive values", + EnvVar: "BUILDKITE_REDACTED_VARS", + }, + cli.StringFlag{ + Name: "tracing-backend", + Usage: "The name of the tracing backend to use.", + EnvVar: "BUILDKITE_TRACING_BACKEND", + Value: "", + }, + cli.StringFlag{ + Name: "tracing-service-name", + Usage: "Service name to use when reporting traces.", + EnvVar: "BUILDKITE_TRACING_SERVICE_NAME", + Value: "buildkite-agent", + }, + DebugFlag, + LogLevelFlag, + ExperimentsFlag, + ProfileFlag, +} + +func execJobAction(c *cli.Context) { + // The configuration will be loaded into this struct + cfg := ExecJobConfig{} + + loader := cliconfig.Loader{CLI: c, Config: &cfg} + warnings, err := loader.Load() + if err != nil { + fmt.Printf("%s", err) + os.Exit(1) + } + + l := CreateLogger(&cfg) + + // Now that we have a logger, log out the warnings that loading config generated + for _, warning := range warnings { + l.Warn("%s", warning) + } + + // Enable experiments + for _, name := range cfg.Experiments { + experiments.Enable(name) + } + + // Handle profiling flag + done := HandleProfileFlag(l, cfg) + defer done() + + // Turn of PTY support if we're on Windows + runInPty := cfg.PTY + if runtime.GOOS == "windows" { + runInPty = false + } + + // Validate phases + for _, phase := range cfg.Phases { + switch phase { + case "plugin", "checkout", "command": + // Valid phase + default: + l.Fatal("Invalid phase %q", phase) + } + } + + cancelSig, err := process.ParseSignal(cfg.CancelSignal) + if err != nil { + l.Fatal("Failed to parse cancel-signal: %v", err) + } + + // Configure the job executor + executor := job.NewExecutor(job.Config{ + AgentName: cfg.AgentName, + ArtifactUploadDestination: cfg.ArtifactUploadDestination, + AutomaticArtifactUploadPaths: cfg.AutomaticArtifactUploadPaths, + BinPath: cfg.BinPath, + Branch: cfg.Branch, + BuildPath: cfg.BuildPath, + CancelSignal: cancelSig, + CleanCheckout: cfg.CleanCheckout, + Command: cfg.Command, + CommandEval: cfg.CommandEval, + Commit: cfg.Commit, + Debug: cfg.Debug, + GitCheckoutFlags: cfg.GitCheckoutFlags, + GitCleanFlags: cfg.GitCleanFlags, + GitCloneFlags: cfg.GitCloneFlags, + GitCloneMirrorFlags: cfg.GitCloneMirrorFlags, + GitFetchFlags: cfg.GitFetchFlags, + GitMirrorsLockTimeout: cfg.GitMirrorsLockTimeout, + GitMirrorsPath: cfg.GitMirrorsPath, + GitMirrorsSkipUpdate: cfg.GitMirrorsSkipUpdate, + GitSubmodules: cfg.GitSubmodules, + GitSubmoduleCloneConfig: cfg.GitSubmoduleCloneConfig, + HooksPath: cfg.HooksPath, + JobID: cfg.JobID, + LocalHooksEnabled: cfg.LocalHooksEnabled, + OrganizationSlug: cfg.OrganizationSlug, + Phases: cfg.Phases, + PipelineProvider: cfg.PipelineProvider, + PipelineSlug: cfg.PipelineSlug, + PluginValidation: cfg.PluginValidation, + Plugins: cfg.Plugins, + PluginsEnabled: cfg.PluginsEnabled, + PluginsAlwaysCloneFresh: cfg.PluginsAlwaysCloneFresh, + PluginsPath: cfg.PluginsPath, + PullRequest: cfg.PullRequest, + Queue: cfg.Queue, + RedactedVars: cfg.RedactedVars, + RefSpec: cfg.RefSpec, + Repository: cfg.Repository, + RunInPty: runInPty, + SSHKeyscan: cfg.SSHKeyscan, + Shell: cfg.Shell, + Tag: cfg.Tag, + TracingBackend: cfg.TracingBackend, + TracingServiceName: cfg.TracingServiceName, + }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + signals := make(chan os.Signal, 1) + signal.Notify(signals, os.Interrupt, + syscall.SIGHUP, + syscall.SIGTERM, + syscall.SIGINT, + syscall.SIGQUIT) + defer signal.Stop(signals) + + var ( + cancelled bool + received os.Signal + signalMu sync.Mutex + ) + + // Listen for signals in the background and cancel the job execution + go func() { + sig := <-signals + signalMu.Lock() + defer signalMu.Unlock() + + // Cancel the job execution + executor.Cancel() + + // Track the state and signal used + cancelled = true + received = sig + + // Remove our signal handler so subsequent signals kill + signal.Stop(signals) + }() + + // Run the job and get the exit code + exitCode := executor.Run(ctx) + + signalMu.Lock() + defer signalMu.Unlock() + + // If cancelled and our child process returns a non-zero, we should terminate + // ourselves with the same signal so that our caller can detect and handle appropriately + if cancelled && runtime.GOOS != "windows" { + p, err := os.FindProcess(os.Getpid()) + if err != nil { + l.Error("Failed to find current process: %v", err) + } + + l.Debug("Terminating job execution after cancellation with %v", received) + err = p.Signal(received) + if err != nil { + l.Error("Failed to signal self: %v", err) + } + } + + os.Exit(exitCode) +} + +var ( + BootstrapCommand = genBootstrap() + ExecJobCommand = genExecJob() +) + +func genBootstrap() cli.Command { + var help strings.Builder + help.WriteString("⚠️ ⚠️ ⚠️\n") + help.WriteString("DEPRECATED: Use `buildkite-agent exec-job` instead\n") + help.WriteString("⚠️ ⚠️ ⚠️\n\n") + err := execJobHelpTpl.Execute(&help, "bootstrap") + if err != nil { + // This can only hapen if we've mangled the template or its parsing + // and will be caught by tests and local usage + // (famous last words) + panic(err) + } + + return cli.Command{ + Name: "bootstrap", + Usage: "[DEPRECATED] Run a Buildkite job locally", + Description: help.String(), + Flags: execJobFlags, + Action: execJobAction, + } +} + +func genExecJob() cli.Command { + var help strings.Builder + err := execJobHelpTpl.Execute(&help, "exec-job") + if err != nil { + // This can only hapen if we've mangled the template or its parsing + // and will be caught by tests and local usage + // (famous last words) + panic(err) + } + + return cli.Command{ + Name: "exec-job", + Usage: "Run a Buildkite job locally", + Description: help.String(), + Flags: execJobFlags, + Action: func(c *cli.Context) { + fmt.Println("⚠️ WARNING ⚠️") + fmt.Println("This command (`buildkite-agent bootstrap`) is deprecated and will be removed in a future release") + fmt.Println("Please use `buildkite-agent exec-job` instead.") + fmt.Println("") + execJobAction(c) + }, + } +} diff --git a/hook/scriptwrapper.go b/hook/scriptwrapper.go index 6136923535..8f2827e795 100644 --- a/hook/scriptwrapper.go +++ b/hook/scriptwrapper.go @@ -3,6 +3,7 @@ package hook import ( "bytes" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -78,7 +79,7 @@ func (e *HookExitError) Error() string { type scriptWrapperOpt func(*ScriptWrapper) -// Hooks get "sourced" into the bootstrap in the sense that they get the +// Hooks get "sourced" into job execution in the sense that they get the // environment set for them and then we capture any extra environment variables // that are exported in the script. @@ -86,7 +87,7 @@ type scriptWrapperOpt func(*ScriptWrapper) // before it finishes, so we've got an awesome (ugly) hack to get around this. // We write the ENV to file, run the hook and then write the ENV back to another file. // Then we can use the diff of the two to figure out what changes to make to the -// bootstrap. Horrible, but effective. +// job executor. Horrible, but effective. // ScriptWrapper wraps a hook script with env collection and then provides // a way to get the difference between the environment before the hook is run and @@ -123,14 +124,14 @@ func NewScriptWrapper(opts ...scriptWrapperOpt) (*ScriptWrapper, error) { } if wrap.hookPath == "" { - return nil, fmt.Errorf("Hook path was not provided") + return nil, errors.New("hook path was not provided") } var err error var isBashHook bool var isPwshHook bool - scriptFileName := "buildkite-agent-bootstrap-hook-runner" + scriptFileName := "buildkite-agent-job-exec-hook-runner" isWindows := wrap.os == "windows" // we use bash hooks for scripts with no extension, otherwise on windows @@ -153,7 +154,7 @@ func NewScriptWrapper(opts ...scriptWrapperOpt) (*ScriptWrapper, error) { // We'll pump the ENV before the hook into this temp file wrap.beforeEnvFile, err = shell.TempFileWithExtension( - "buildkite-agent-bootstrap-hook-env-before", + "buildkite-agent-job-exec-hook-env-before", ) if err != nil { return nil, err @@ -162,7 +163,7 @@ func NewScriptWrapper(opts ...scriptWrapperOpt) (*ScriptWrapper, error) { // We'll then pump the ENV _after_ the hook into this temp file wrap.afterEnvFile, err = shell.TempFileWithExtension( - "buildkite-agent-bootstrap-hook-env-after", + "buildkite-agent-job-exec-hook-env-after", ) if err != nil { return nil, err diff --git a/job/executor.go b/job/executor.go index a7b1b77e9c..4a30d91fe3 100644 --- a/job/executor.go +++ b/job/executor.go @@ -56,8 +56,8 @@ type Executor struct { cancelCh chan struct{} } -// New returns a new Executor instance -func New(conf Config) *Executor { +// NewExecutor returns a new Executor instance +func NewExecutor(conf Config) *Executor { return &Executor{ Config: conf, cancelCh: make(chan struct{}), diff --git a/job/executor_test.go b/job/executor_test.go index 9f72c038b0..cb72c0b39f 100644 --- a/job/executor_test.go +++ b/job/executor_test.go @@ -83,7 +83,7 @@ func TestStartTracing_NoTracingBackend(t *testing.T) { var err error // When there's no tracing backend, the tracer should be a no-op. - b := New(Config{}) + b := NewExecutor(Config{}) oriCtx := context.Background() b.shell, err = shell.New() @@ -104,7 +104,7 @@ func TestStartTracing_Datadog(t *testing.T) { // With the Datadog tracing backend, the global tracer should be from Datadog. cfg := Config{TracingBackend: "datadog"} - b := New(cfg) + b := NewExecutor(cfg) oriCtx := context.Background() b.shell, err = shell.New() diff --git a/job/integration/executor_tester.go b/job/integration/executor_tester.go index 41de084a14..0440653406 100644 --- a/job/integration/executor_tester.go +++ b/job/integration/executor_tester.go @@ -76,7 +76,7 @@ func NewExecutorTester() (*ExecutorTester, error) { bt := &ExecutorTester{ Name: os.Args[0], - Args: []string{"bootstrap"}, + Args: []string{"exec-job"}, Repo: repo, Env: []string{ "HOME=" + homeDir, diff --git a/job/integration/main_test.go b/job/integration/main_test.go index 86c01a46f7..d5fcd86469 100644 --- a/job/integration/main_test.go +++ b/job/integration/main_test.go @@ -13,13 +13,13 @@ import ( ) func TestMain(m *testing.M) { - // If we are passed "bootstrap", execute like the bootstrap cli - if len(os.Args) > 1 && os.Args[1] == "bootstrap" { + // If we are passed "exec-job", execute like the exec-job cli + if len(os.Args) > 1 && os.Args[1] == "exec-job" { app := cli.NewApp() app.Name = "buildkite-agent" app.Version = version.Version() app.Commands = []cli.Command{ - clicommand.BootstrapCommand, + clicommand.ExecJobCommand, } if err := app.Run(os.Args); err != nil { diff --git a/main.go b/main.go index a19de768af..516ae04abe 100644 --- a/main.go +++ b/main.go @@ -122,6 +122,7 @@ func main() { }, }, clicommand.BootstrapCommand, + clicommand.ExecJobCommand, } app.ErrWriter = os.Stderr diff --git a/packaging/docker/alpine-k8s/buildkite-agent.cfg b/packaging/docker/alpine-k8s/buildkite-agent.cfg index 4899c2c8de..5073031373 100644 --- a/packaging/docker/alpine-k8s/buildkite-agent.cfg +++ b/packaging/docker/alpine-k8s/buildkite-agent.cfg @@ -25,10 +25,10 @@ name="%hostname-%spawn" # Include the host's Google Cloud instance labels as tags # tags-from-gcp-labels=true -# Path to a custom bootstrap command to run. By default this is `buildkite-agent bootstrap`. +# Path to a custom job execution command to run. By default this is `buildkite-agent exec-job`. # This allows you to override the entire execution of a job. Generally you should use hooks instead! # See https://buildkite.com/docs/agent/hooks -# bootstrap-script="" +# job-executor-script="" # Path to where the builds will run from build-path="/buildkite/builds" diff --git a/packaging/docker/alpine/buildkite-agent.cfg b/packaging/docker/alpine/buildkite-agent.cfg index 4899c2c8de..5073031373 100644 --- a/packaging/docker/alpine/buildkite-agent.cfg +++ b/packaging/docker/alpine/buildkite-agent.cfg @@ -25,10 +25,10 @@ name="%hostname-%spawn" # Include the host's Google Cloud instance labels as tags # tags-from-gcp-labels=true -# Path to a custom bootstrap command to run. By default this is `buildkite-agent bootstrap`. +# Path to a custom job execution command to run. By default this is `buildkite-agent exec-job`. # This allows you to override the entire execution of a job. Generally you should use hooks instead! # See https://buildkite.com/docs/agent/hooks -# bootstrap-script="" +# job-executor-script="" # Path to where the builds will run from build-path="/buildkite/builds" diff --git a/packaging/docker/sidecar/buildkite-agent.cfg b/packaging/docker/sidecar/buildkite-agent.cfg index 4899c2c8de..5073031373 100644 --- a/packaging/docker/sidecar/buildkite-agent.cfg +++ b/packaging/docker/sidecar/buildkite-agent.cfg @@ -25,10 +25,10 @@ name="%hostname-%spawn" # Include the host's Google Cloud instance labels as tags # tags-from-gcp-labels=true -# Path to a custom bootstrap command to run. By default this is `buildkite-agent bootstrap`. +# Path to a custom job execution command to run. By default this is `buildkite-agent exec-job`. # This allows you to override the entire execution of a job. Generally you should use hooks instead! # See https://buildkite.com/docs/agent/hooks -# bootstrap-script="" +# job-executor-script="" # Path to where the builds will run from build-path="/buildkite/builds" diff --git a/packaging/docker/ubuntu-18.04/buildkite-agent.cfg b/packaging/docker/ubuntu-18.04/buildkite-agent.cfg index 4899c2c8de..5073031373 100644 --- a/packaging/docker/ubuntu-18.04/buildkite-agent.cfg +++ b/packaging/docker/ubuntu-18.04/buildkite-agent.cfg @@ -25,10 +25,10 @@ name="%hostname-%spawn" # Include the host's Google Cloud instance labels as tags # tags-from-gcp-labels=true -# Path to a custom bootstrap command to run. By default this is `buildkite-agent bootstrap`. +# Path to a custom job execution command to run. By default this is `buildkite-agent exec-job`. # This allows you to override the entire execution of a job. Generally you should use hooks instead! # See https://buildkite.com/docs/agent/hooks -# bootstrap-script="" +# job-executor-script="" # Path to where the builds will run from build-path="/buildkite/builds" diff --git a/packaging/docker/ubuntu-20.04/buildkite-agent.cfg b/packaging/docker/ubuntu-20.04/buildkite-agent.cfg index 4899c2c8de..5073031373 100644 --- a/packaging/docker/ubuntu-20.04/buildkite-agent.cfg +++ b/packaging/docker/ubuntu-20.04/buildkite-agent.cfg @@ -25,10 +25,10 @@ name="%hostname-%spawn" # Include the host's Google Cloud instance labels as tags # tags-from-gcp-labels=true -# Path to a custom bootstrap command to run. By default this is `buildkite-agent bootstrap`. +# Path to a custom job execution command to run. By default this is `buildkite-agent exec-job`. # This allows you to override the entire execution of a job. Generally you should use hooks instead! # See https://buildkite.com/docs/agent/hooks -# bootstrap-script="" +# job-executor-script="" # Path to where the builds will run from build-path="/buildkite/builds" diff --git a/packaging/github/linux/buildkite-agent.cfg b/packaging/github/linux/buildkite-agent.cfg index affe7792fe..88d0baec6b 100644 --- a/packaging/github/linux/buildkite-agent.cfg +++ b/packaging/github/linux/buildkite-agent.cfg @@ -25,10 +25,10 @@ name="%hostname-%spawn" # Include the host's Google Cloud instance labels as tags # tags-from-gcp-labels=true -# Path to a custom bootstrap command to run. By default this is `buildkite-agent bootstrap`. +# Path to a custom job execution command to run. By default this is `buildkite-agent exec-job`. # This allows you to override the entire execution of a job. Generally you should use hooks instead! # See https://buildkite.com/docs/agent/hooks -# bootstrap-script="" +# job-executor-script="" # Path to where the builds will run from build-path="$HOME/.buildkite-agent/builds" diff --git a/packaging/github/windows/buildkite-agent.cfg b/packaging/github/windows/buildkite-agent.cfg index f98f8fed4d..20c4c7ba6a 100644 --- a/packaging/github/windows/buildkite-agent.cfg +++ b/packaging/github/windows/buildkite-agent.cfg @@ -13,10 +13,10 @@ name="%hostname-%spawn" # Tags for the agent (default is "queue=default") # tags="key1=val2,key2=val2" -# Path to a custom bootstrap command to run. By default this is `buildkite-agent bootstrap`. +# Path to a custom job execution command to run. By default this is `buildkite-agent exec-job`. # This allows you to override the entire execution of a job. Generally you should use hooks instead! # See https://buildkite.com/docs/agent/hooks -# bootstrap-script="" +# job-executor-script="" # Path to where the builds will run from build-path="C:\buildkite-agent\builds" diff --git a/utils/path_test.go b/utils/path_test.go index 7d1742095f..ddf45fab66 100644 --- a/utils/path_test.go +++ b/utils/path_test.go @@ -47,9 +47,9 @@ func TestNormalizingCommands(t *testing.T) { usr, err := user.Current() assert.NoError(t, err) - c, err := NormalizeCommand(filepath.Join("~/", "buildkite-agent", "bootstrap.sh")) + c, err := NormalizeCommand(filepath.Join("~/", "buildkite-agent", "exec-job")) assert.NoError(t, err) - assert.Equal(t, filepath.Join(usr.HomeDir, "buildkite-agent", "bootstrap.sh"), c) + assert.Equal(t, filepath.Join(usr.HomeDir, "buildkite-agent", "exec-job"), c) c, err = NormalizeCommand("cat test.log") assert.NoError(t, err)