From fd6f48302b3759127ff5df7e9d0ae25580fbc582 Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Thu, 30 May 2024 14:39:20 +1000 Subject: [PATCH 1/4] Specify go version in go.mod --- go.mod | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.mod b/go.mod index b0e8b67..5c15603 100644 --- a/go.mod +++ b/go.mod @@ -1 +1,3 @@ module github.com/buildkite/interpolate + +go 1.22.3 From 642e81d4796c7260ee4603ca859b483d1023b184 Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Thu, 30 May 2024 14:44:00 +1000 Subject: [PATCH 2/4] Add buildkite pipeline wow, embarrassing that that was missing, huh? --- .buildkite/pipeline.yml | 26 ++++++++++++++++++++++++++ .buildkite/steps/lint.sh | 25 +++++++++++++++++++++++++ .buildkite/steps/test.sh | 11 +++++++++++ README.md | 1 + 4 files changed, 63 insertions(+) create mode 100644 .buildkite/pipeline.yml create mode 100755 .buildkite/steps/lint.sh create mode 100755 .buildkite/steps/test.sh diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 0000000..2b9c5ac --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,26 @@ +steps: + - name: ":go::robot_face: Lint" + key: lint + command: .buildkite/steps/lint.sh + plugins: + - docker#v5.11.0: + image: "golang:1.22" + + - name: ":go::test_tube: Test" + key: test + command: ".buildkite/steps/test.sh" + artifact_paths: junit-*.xml + plugins: + - docker#v5.11.0: + image: "golang:1.22" + propagate-environment: true + - artifacts#v1.9.0: + upload: "cover.{html,out}" + + - label: ":writing_hand: Annotate with Test Failures" + key: annotate + depends_on: test + allow_dependency_failure: true + plugins: + - junit-annotate#v1.6.0: + artifacts: junit-*.xml diff --git a/.buildkite/steps/lint.sh b/.buildkite/steps/lint.sh new file mode 100755 index 0000000..8224da4 --- /dev/null +++ b/.buildkite/steps/lint.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -Eeufo pipefail + +echo --- :go: Checking go mod tidiness +go mod tidy +if ! git diff --no-ext-diff --exit-code; then + echo ^^^ +++ + echo "The go.mod or go.sum files are out of sync with the source code" + echo "Please run \`go mod tidy\` locally, and commit the result." + + exit 1 +fi + +echo --- :go: Checking go formatting +gofmt -w . +if ! git diff --no-ext-diff --exit-code; then + echo ^^^ +++ + echo "Files have not been formatted with gofmt." + echo "Fix this by running \`go fmt ./...\` locally, and committing the result." + + exit 1 +fi + +echo +++ Everything is clean and tidy! 🎉 diff --git a/.buildkite/steps/test.sh b/.buildkite/steps/test.sh new file mode 100755 index 0000000..460daba --- /dev/null +++ b/.buildkite/steps/test.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +go install gotest.tools/gotestsum@v1.8.0 + +echo '+++ Running tests' +gotestsum --junitfile "junit-${BUILDKITE_JOB_ID}.xml" -- -count=1 -coverprofile=cover.out -race "$@" ./... + +echo 'Producing coverage report' +go tool cover -html cover.out -o cover.html diff --git a/README.md b/README.md index ec2790e..d5167bd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ Interpolate =========== +[![Build status](https://badge.buildkite.com/3d9081230a965a20d7b68d66564644febdada7f968cc6f9c9c.svg)](https://buildkite.com/buildkite/interpolate) [![GoDoc](https://godoc.org/github.com/buildkite/interpolate?status.svg)](https://godoc.org/github.com/buildkite/interpolate) A golang library for parameter expansion (like `${BLAH}` or `$BLAH`) in strings from environment variables. An implementation of [POSIX Parameter Expansion](http://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_06_02), plus some other basic operations that you'd expect in a shell scripting environment [like bash](https://www.gnu.org/software/bash/manual/html_node/Shell-Parameter-Expansion.html). From d56c9579c3145c8e5d00948fdb27ac337c5a1f85 Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Thu, 30 May 2024 15:24:27 +1000 Subject: [PATCH 3/4] Include escaped expansions in calls to Identifiers It's useful for consumers of this libary to be able to know if their input contains things that would be interpolated, and if so, which things. Currently, this is handled by the `interpolate.Identifiers()` function, which returns a list of the interpolate-able identifiers that `interpolate.Interpolate()` would replace. However, this library makes one further modification to its input: when escaped expansions are included (ie, $$MY_VAR or \$MY_VAR), they're de-escaped in the output. This is working as intended, but the `interpolate.Identifiers` function doesn't include these escaped interpolations, which is a bit confusing when you're using the `Identifiers` function to determine "will running Interpolate() change the input?" To remedy this, this PR updates the parse to handle escaped interpolations a bit differently, and treat them as a kind of expansion that doesn't take any input, and de-escapes its input. There's no functionality change, but escaped interpolations now show up in the `Identifiers()` function --- README.md | 5 ++++- interpolate.go | 13 +++++++++++ interpolate_test.go | 4 ++++ parser.go | 55 +++++++++++++++++++++++++++++++++------------ parser_test.go | 10 +++++---- 5 files changed, 68 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index d5167bd..114e215 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,10 @@ func main() {
Use the substring of parameter after offset of given length. A negative offset must be separated from the colon with a space, and will select from the end of the string. If the offset is out of bounds, an empty string will be substituted. If the length is greater than the length then the entire string will be returned.
${parameter:?[word]}
-
Indicate Error if Null or Unset. If parameter is unset or null, the expansion of word (or a message indicating it is unset if word is omitted) shall be returned as an error.
+
Indicate Error if Null or Unset. If parameter is unset or null, the expansion of word (or a message indicating it is unset if word is omitted) shall be returned as an error.
+ +
$$parameter or \$parameter or $${expression} or \${expression}
+
An escaped interpolation. Will not be interpolated, but will be unescaped by a call to interpolate.Interpolate()
## License diff --git a/interpolate.go b/interpolate.go index 90d5ee3..7a24fd8 100644 --- a/interpolate.go +++ b/interpolate.go @@ -82,6 +82,19 @@ func (e UnsetValueExpansion) Expand(env Env) (string, error) { return val, nil } +// EscapedExpansion is an expansion that is delayed until later on (usually by a later process) +type EscapedExpansion struct { + Identifier string +} + +func (e EscapedExpansion) Identifiers() []string { + return []string{"$" + e.Identifier} +} + +func (e EscapedExpansion) Expand(Env) (string, error) { + return "$" + e.Identifier, nil +} + // SubstringExpansion returns a substring (or slice) of the env type SubstringExpansion struct { Identifier string diff --git a/interpolate_test.go b/interpolate_test.go index 6a2e4c0..1bba0b5 100644 --- a/interpolate_test.go +++ b/interpolate_test.go @@ -260,6 +260,8 @@ func TestEscapingVariables(t *testing.T) { {`Do this \$ESCAPE_PARTY`, `Do this $ESCAPE_PARTY`}, {`Do this $${SUCH_ESCAPE}`, `Do this ${SUCH_ESCAPE}`}, {`Do this \${SUCH_ESCAPE}`, `Do this ${SUCH_ESCAPE}`}, + {`Do this $${SUCH_ESCAPE:-$OTHERWISE}`, `Do this ${SUCH_ESCAPE:-$OTHERWISE}`}, + {`Do this \${SUCH_ESCAPE:-$OTHERWISE}`, `Do this ${SUCH_ESCAPE:-$OTHERWISE}`}, } { result, err := interpolate.Interpolate(nil, tc.Str) if err != nil { @@ -279,11 +281,13 @@ func TestExtractingIdentifiers(t *testing.T) { {`Hello ${REQUIRED_VAR?}`, []string{`REQUIRED_VAR`}}, {`${LLAMAS:-${ROCK:-true}}`, []string{`LLAMAS`, `ROCK`}}, {`${BUILDKITE_COMMIT:0}`, []string{`BUILDKITE_COMMIT`}}, + {`$BUILDKITE_COMMIT hello there $$DOUBLE_DOLLAR \$ESCAPED_DOLLAR`, []string{`BUILDKITE_COMMIT`, `$DOUBLE_DOLLAR`, `$ESCAPED_DOLLAR`}}, } { id, err := interpolate.Identifiers(tc.Str) if err != nil { t.Fatal(err) } + if !reflect.DeepEqual(id, tc.Identifiers) { t.Fatalf("Test %q should have identifiers %v, got %v", tc.Str, tc.Identifiers, id) } diff --git a/parser.go b/parser.go index fa7e843..52954c2 100644 --- a/parser.go +++ b/parser.go @@ -23,18 +23,20 @@ import ( // and structs named the same reading the string bite by bite (peekRune and nextRune) /* -EscapedBackslash = "\\" -EscapedDollar = ( "\$" | "$$") -Identifier = letter { letters | digit | "_" } -Expansion = "$" ( Identifier | Brace ) -Brace = "{" Identifier [ Identifier BraceOperation ] "}" -Text = { EscapedBackslash | EscapedDollar | all characters except "$" } -Expression = { Text | Expansion } -EmptyValue = ":-" { Expression } -UnsetValue = "-" { Expression } -Substring = ":" number [ ":" number ] -Required = "?" { Expression } -Operation = EmptyValue | UnsetValue | Substring | Required +EscapedBackslash = "\\" +Identifier = letter { letters | digit | "_" } +EscapedDollar = ( "\$" | "$$" ) +EscapedExpansion = EscapedDollar ( Identifier | Brace ) +UnescapedExpansion = "$" ( Identifier | Brace ) +Expansion = UnescapedExpansion | EscapedExpansion +Brace = "{" Identifier [ Identifier BraceOperation ] "}" +Text = { EscapedBackslash | EscapedDollar | all characters except "$" } +Expression = { Text | Expansion } +EmptyValue = ":-" { Expression } +UnsetValue = "-" { Expression } +Substring = ":" number [ ":" number ] +Required = "?" { Expression } +Operation = EmptyValue | UnsetValue | Substring | Required */ const ( @@ -75,9 +77,17 @@ func (p *Parser) parseExpression(stop ...rune) (Expression, error) { p.pos += 2 expr = append(expr, ExpressionItem{Text: `\\`}) continue - } else if strings.HasPrefix(p.input[p.pos:], `\$`) || strings.HasPrefix(p.input[p.pos:], `$$`) { + } + + if strings.HasPrefix(p.input[p.pos:], `\$`) || strings.HasPrefix(p.input[p.pos:], `$$`) { p.pos += 2 - expr = append(expr, ExpressionItem{Text: `$`}) + + ee, err := p.parseEscapedExpansion() + if err != nil { + return nil, err + } + + expr = append(expr, ExpressionItem{Expansion: ee}) continue } @@ -112,6 +122,23 @@ func (p *Parser) parseExpression(stop ...rune) (Expression, error) { return expr, nil } +func (p *Parser) parseEscapedExpansion() (Expansion, error) { + // if it's an escaped brace expansion, (eg $${MY_COOL_VAR:-5}) consume text until the close brace + if c := p.peekRune(); c == '{' { + id := p.scanUntil(func(r rune) bool { return r == '}' }) + id = id + string(p.nextRune()) // we know that the next rune is a close brace, chuck it on the end + return EscapedExpansion{Identifier: id}, nil + } + + // otherwise, it's an escaped identifier (eg $$MY_COOL_VAR) + id, err := p.scanIdentifier() + if err != nil { + return nil, err + } + + return EscapedExpansion{Identifier: id}, nil +} + func (p *Parser) parseExpansion() (Expansion, error) { if c := p.nextRune(); c != '$' { return nil, fmt.Errorf("Expected expansion to start with $, got %c", c) diff --git a/parser_test.go b/parser_test.go index 3254cf4..8f8cb4f 100644 --- a/parser_test.go +++ b/parser_test.go @@ -73,8 +73,7 @@ func TestParser(t *testing.T) { { String: `\${HELLO_WORLD-blah}`, Expected: []interpolate.ExpressionItem{ - {Text: `$`}, - {Text: `{HELLO_WORLD-blah}`}, + {Expansion: interpolate.EscapedExpansion{Identifier: "{HELLO_WORLD-blah}"}}, }, }, { @@ -82,8 +81,7 @@ func TestParser(t *testing.T) { Expected: []interpolate.ExpressionItem{ {Text: `Test `}, {Text: `\\`}, - {Text: `$`}, - {Text: `{HELLO_WORLD-blah}`}, + {Expansion: interpolate.EscapedExpansion{Identifier: "{HELLO_WORLD-blah}"}}, }, }, { @@ -167,6 +165,10 @@ func TestParser(t *testing.T) { {Text: `echo hello world)`}, }, }, + { + String: "$$MOUNTAIN", + Expected: []interpolate.ExpressionItem{{Expansion: interpolate.EscapedExpansion{Identifier: "MOUNTAIN"}}}, + }, } for _, tc := range testCases { From b90a3c0d6498118845634a119f2f8f3b2f19f07c Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Thu, 30 May 2024 16:31:08 +1000 Subject: [PATCH 4/4] Parallise tests --- interpolate_test.go | 30 ++++++++++++++++++++++++++++++ parser_test.go | 4 ++++ 2 files changed, 34 insertions(+) diff --git a/interpolate_test.go b/interpolate_test.go index 1bba0b5..9c996d3 100644 --- a/interpolate_test.go +++ b/interpolate_test.go @@ -24,6 +24,8 @@ func ExampleInterpolate() { } func TestBasicInterpolation(t *testing.T) { + t.Parallel() + environ := interpolate.NewMapEnv(map[string]string{ "TEST1": "A test", "TEST2": "Another", @@ -49,6 +51,8 @@ func TestBasicInterpolation(t *testing.T) { {`${TEST4}`, "Only one level of $TEST3 interpolation"}, } { t.Run(tc.Str, func(t *testing.T) { + t.Parallel() + result, err := interpolate.Interpolate(environ, tc.Str) if err != nil { t.Fatal(err) @@ -61,6 +65,8 @@ func TestBasicInterpolation(t *testing.T) { } func TestNestedInterpolation(t *testing.T) { + t.Parallel() + environ := interpolate.NewMapEnv(map[string]string{ "TEST1": "A test", "TEST2": "Another", @@ -77,6 +83,8 @@ func TestNestedInterpolation(t *testing.T) { {`${TEST5:-Some text ${TEST2:-$TEST1} with $TEST3}`, "Some text Another with Llamas"}, } { t.Run(tc.Str, func(t *testing.T) { + t.Parallel() + result, err := interpolate.Interpolate(environ, tc.Str) if err != nil { t.Fatal(err) @@ -89,6 +97,8 @@ func TestNestedInterpolation(t *testing.T) { } func TestIgnoresParentheses(t *testing.T) { + t.Parallel() + for _, str := range []string{ `$(echo hello world)`, `testing $(echo hello world)`, @@ -105,6 +115,8 @@ func TestIgnoresParentheses(t *testing.T) { } func TestVariablesMustStartWithLetters(t *testing.T) { + t.Parallel() + for _, str := range []string{ `$1 burgers`, `$99bottles`, @@ -119,6 +131,8 @@ func TestVariablesMustStartWithLetters(t *testing.T) { } func TestMissingParameterValuesReturnEmptyStrings(t *testing.T) { + t.Parallel() + for _, str := range []string{ `$BUILDKITE_COMMIT`, `${BUILDKITE_COMMIT}`, @@ -128,6 +142,8 @@ func TestMissingParameterValuesReturnEmptyStrings(t *testing.T) { `${BUILDKITE_COMMIT:7:14}`, } { t.Run(str, func(t *testing.T) { + t.Parallel() + result, err := interpolate.Interpolate(nil, str) if err != nil { t.Fatal(err) @@ -140,6 +156,8 @@ func TestMissingParameterValuesReturnEmptyStrings(t *testing.T) { } func TestSubstringsWithOffsets(t *testing.T) { + t.Parallel() + environ := interpolate.NewMapEnv(map[string]string{"BUILDKITE_COMMIT": "1adf998e39f647b4b25842f107c6ed9d30a3a7c7"}) for _, tc := range []struct { @@ -171,6 +189,8 @@ func TestSubstringsWithOffsets(t *testing.T) { {`${BUILDKITE_COMMIT:7:-128}`, ``}, } { t.Run(tc.Str, func(t *testing.T) { + t.Parallel() + result, err := interpolate.Interpolate(environ, tc.Str) if err != nil { t.Fatal(err) @@ -183,6 +203,8 @@ func TestSubstringsWithOffsets(t *testing.T) { } func TestInterpolateIsntGreedy(t *testing.T) { + t.Parallel() + environ := interpolate.NewMapEnv(map[string]string{ "BUILDKITE_COMMIT": "cfeeee3fa7fa1a6311723f5cbff95b738ec6e683", "BUILDKITE_PARALLEL_JOB": "456", @@ -207,6 +229,8 @@ func TestInterpolateIsntGreedy(t *testing.T) { } func TestDefaultValues(t *testing.T) { + t.Parallel() + environ := interpolate.NewMapEnv(map[string]string{ "DAY": "Blarghday", "EMPTY_DAY": "", @@ -236,6 +260,8 @@ func TestDefaultValues(t *testing.T) { } func TestRequiredVariables(t *testing.T) { + t.Parallel() + for _, tc := range []struct { Str string ExpectedErr string @@ -252,6 +278,8 @@ func TestRequiredVariables(t *testing.T) { } func TestEscapingVariables(t *testing.T) { + t.Parallel() + for _, tc := range []struct { Str string Expected string @@ -274,6 +302,8 @@ func TestEscapingVariables(t *testing.T) { } func TestExtractingIdentifiers(t *testing.T) { + t.Parallel() + for _, tc := range []struct { Str string Identifiers []string diff --git a/parser_test.go b/parser_test.go index 8f8cb4f..747ccdf 100644 --- a/parser_test.go +++ b/parser_test.go @@ -8,6 +8,8 @@ import ( ) func TestParser(t *testing.T) { + t.Parallel() + var testCases = []struct { String string Expected []interpolate.ExpressionItem @@ -173,6 +175,8 @@ func TestParser(t *testing.T) { for _, tc := range testCases { t.Run(tc.String, func(t *testing.T) { + t.Parallel() + actual, err := interpolate.NewParser(tc.String).Parse() if err != nil { t.Fatal(err)