From cda7f392abf66f821b0f9bdc8c3bf541cca03803 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Fri, 4 Apr 2025 12:23:49 -0400 Subject: [PATCH 1/3] task env: add NOMAD_UNIX_ADDR var for easier task setup when using workload identity and nomad CLI --- .changelog/25598.txt | 3 ++ client/taskenv/env.go | 6 ++++ client/taskenv/env_test.go | 8 +++-- e2e/workload_id/input/api-nomad-cli.nomad.hcl | 32 +++++++++++++++++++ e2e/workload_id/taskapi_test.go | 17 ++++++++++ website/content/docs/commands/index.mdx | 7 ++-- website/content/partials/envvars.mdx | 2 ++ 7 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 .changelog/25598.txt create mode 100644 e2e/workload_id/input/api-nomad-cli.nomad.hcl diff --git a/.changelog/25598.txt b/.changelog/25598.txt new file mode 100644 index 00000000000..2400d3aa417 --- /dev/null +++ b/.changelog/25598.txt @@ -0,0 +1,3 @@ +```release-note:improvement +task environment: add NOMAD_UNIX_ADDR env var when using workload identity +``` diff --git a/client/taskenv/env.go b/client/taskenv/env.go index 9a8ab9e9711..f7434f40153 100644 --- a/client/taskenv/env.go +++ b/client/taskenv/env.go @@ -98,6 +98,10 @@ const ( HostAddrPrefix = "NOMAD_HOST_ADDR_" + // UnixAddr is the task api unix socket, in the appropriate format + // for use in a NOMAD_ADDR (i.e. prefixed with "unix:") + UnixAddr = "NOMAD_UNIX_ADDR" + // IpPrefix is the prefix for passing the host IP of a port allocation // to a task. IpPrefix = "NOMAD_IP_" @@ -620,10 +624,12 @@ func (b *Builder) buildEnv(allocDir, localDir, secretsDir string, // Build the Nomad Workload Token if b.workloadTokenDefault != "" { envMap[WorkloadToken] = b.workloadTokenDefault + envMap[UnixAddr] = "unix:" + filepath.Join(secretsDir, "api.sock") } for name, token := range b.workloadTokens { envMap[WorkloadToken+"_"+name] = token + envMap[UnixAddr] = "unix:" + filepath.Join(secretsDir, "api.sock") } // Copy and interpolate task meta diff --git a/client/taskenv/env_test.go b/client/taskenv/env_test.go index 8df01fb7262..5297e11ed23 100644 --- a/client/taskenv/env_test.go +++ b/client/taskenv/env_test.go @@ -214,7 +214,7 @@ func TestEnvironment_AsList(t *testing.T) { } env := NewBuilder(n, a, task, "global").SetDriverNetwork( &drivers.DriverNetwork{PortMap: map[string]int{"https": 443}}, - ) + ).SetDefaultWorkloadToken("test-wi-token") act := env.Build().List() exp := []string{ @@ -263,6 +263,8 @@ func TestEnvironment_AsList(t *testing.T) { fmt.Sprintf("NOMAD_ALLOC_ID=%s", a.ID), fmt.Sprintf("NOMAD_SHORT_ALLOC_ID=%s", a.ID[:8]), "NOMAD_ALLOC_INDEX=0", + "NOMAD_TOKEN=test-wi-token", + "NOMAD_UNIX_ADDR=unix:api.sock", } sort.Strings(act) sort.Strings(exp) @@ -342,7 +344,7 @@ func TestEnvironment_AllValues(t *testing.T) { } env := NewBuilder(n, a, task, "global").SetDriverNetwork( &drivers.DriverNetwork{PortMap: map[string]int{"https": 443}}, - ) + ).SetDefaultWorkloadToken("test-wi-token") // Setting the network status ensures we trigger the addNomadAllocNetwork // for the test. @@ -453,6 +455,8 @@ func TestEnvironment_AllValues(t *testing.T) { "NOMAD_ALLOC_INTERFACE_admin": "eth0", "NOMAD_ALLOC_IP_admin": "172.26.64.19", "NOMAD_ALLOC_ADDR_admin": "172.26.64.19:9000", + "NOMAD_TOKEN": "test-wi-token", + "NOMAD_UNIX_ADDR": "unix:api.sock", // Env vars from the host. "LC_CTYPE": "C.UTF-8", diff --git a/e2e/workload_id/input/api-nomad-cli.nomad.hcl b/e2e/workload_id/input/api-nomad-cli.nomad.hcl new file mode 100644 index 00000000000..6e0b34239e6 --- /dev/null +++ b/e2e/workload_id/input/api-nomad-cli.nomad.hcl @@ -0,0 +1,32 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +job "task-api-nomad-cli" { + type = "batch" + + group "grp" { + restart { attempts = 0 } + reschedule { attempts = 0 } + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + task "tsk" { + driver = "raw_exec" + config { + command = "bash" + // "|| true" because failure to get a var makes nomad cli exit 1, + // but for this test, "Variable not found" actually indicates successful + // API connection. + args = ["-xc", "echo $NOMAD_ADDR; nomad var get nothing || true"] + } + env { + NOMAD_ADDR = "${NOMAD_UNIX_ADDR}" + } + identity { # creates unix addr + env = true # provides NOMAD_TOKEN + } + } + } +} diff --git a/e2e/workload_id/taskapi_test.go b/e2e/workload_id/taskapi_test.go index e44dd5abe95..18924590454 100644 --- a/e2e/workload_id/taskapi_test.go +++ b/e2e/workload_id/taskapi_test.go @@ -11,6 +11,8 @@ import ( "testing" "github.com/hashicorp/nomad/e2e/e2eutil" + "github.com/hashicorp/nomad/e2e/v3/cluster3" + "github.com/hashicorp/nomad/e2e/v3/jobs3" "github.com/hashicorp/nomad/helper/uuid" "github.com/shoenig/test" "github.com/shoenig/test/must" @@ -26,6 +28,7 @@ func TestTaskAPI(t *testing.T) { t.Run("testTaskAPI_Auth", testTaskAPIAuth) t.Run("testTaskAPI_Windows", testTaskAPIWindows) + t.Run("testTaskAPI_NomadCLI", testTaskAPINomadCLI) } func testTaskAPIAuth(t *testing.T) { @@ -129,3 +132,17 @@ func testTaskAPIWindows(t *testing.T) { must.StrHasSuffix(t, `"ok":true}}`, logs) } + +func testTaskAPINomadCLI(t *testing.T) { + cluster3.Establish(t, + cluster3.LinuxClients(1), + ) + sub, _ := jobs3.Submit(t, + "./input/api-nomad-cli.nomad.hcl", + jobs3.WaitComplete("grp"), + ) + logs := sub.TaskLogs("grp", "tsk") + test.StrContains(t, logs.Stdout, "unix:/") // from `echo $NOMAD_ADDR` + test.StrContains(t, logs.Stdout, "secrets/api.sock") + test.StrContains(t, logs.Stderr, "Variable not found") // api success +} diff --git a/website/content/docs/commands/index.mdx b/website/content/docs/commands/index.mdx index 13bc8b105ed..00783715996 100644 --- a/website/content/docs/commands/index.mdx +++ b/website/content/docs/commands/index.mdx @@ -35,7 +35,8 @@ $ nomad -autocomplete-uninstall Nomad's CLI commands have implied contexts in their naming convention. Because the CLI is most commonly used to manipulate or query jobs, you can assume that any given command is working in that context unless the command name implies -otherwise. For example, the `nomad job run` command runs a new job and the `nomad status` command queries information about existing jobs. Conversely, +otherwise. For example, the `nomad job run` command runs a new job and the +`nomad status` command queries information about existing jobs. Conversely, commands with a prefix in their name likely operate in a different context. Examples include the `nomad agent-info` or `nomad node drain` commands, which operate in the agent or node contexts respectively. @@ -64,7 +65,8 @@ may override these environment variables with individual flags. #### Connection environment variables - `NOMAD_ADDR` - The address of the Nomad server. Defaults to - `http://127.0.0.1:4646`. + `http://127.0.0.1:4646`. For unix sockets, as with the [task API][], + the format is `unix:/path/to/api.sock`. - `NOMAD_REGION` - The region of the Nomad server to forward commands to. Defaults to the Agent's local region @@ -118,4 +120,5 @@ may override these environment variables with individual flags. - `NOMAD_LICENSE` - The Nomad Enterprise license file contents as a string. +[task API]: /nomad/api-docs/task-api [`tls` block in agent configuration]: /nomad/docs/configuration/tls diff --git a/website/content/partials/envvars.mdx b/website/content/partials/envvars.mdx index 1ed32107d46..377c649f84d 100644 --- a/website/content/partials/envvars.mdx +++ b/website/content/partials/envvars.mdx @@ -22,6 +22,7 @@ | `NOMAD_PARENT_CGROUP` | The parent cgroup used to contain task cgroups (Linux only) | | `NOMAD_NAMESPACE` | Namespace in which the allocation is running | | `NOMAD_REGION` | Region in which the allocation is running | +| `NOMAD_UNIX_ADDR` | Use this value as your `NOMAD_ADDR` to use `nomad` CLI with the [task API][]'s unix socket. The value is equivalent to `"unix:${NOMAD_SECRETS_DIR}/api.sock"` | `NOMAD_META_` | The metadata value given by `key` on the task's metadata. Any character in a key other than `[A-Za-z0-9_.]` will be converted to `_`.
**Note:** this is different from [`${meta.}`](/nomad/docs/runtime/interpolation#node-variables-) which are keys in the node's metadata. | | `CONSUL_HTTP_TOKEN` | The tasks' Consul token. See [Consul Integration][consul] documentation for more details. | | `CONSUL_TOKEN` | The tasks' Consul token. See [Consul Integration][consul] documentation for more details. This variable is deprecated and exists only for backwards compatibility. | @@ -68,6 +69,7 @@ names such as `NOMAD_ADDR__