Merge pull request #10 from buildkite/include-escaped-interpolations-…

…in-identifiers-2

Include escaped interpolations in identifiers

.buildkite/pipeline.yml

@@ -0,0 +1,26 @@
+steps:
+  - name: ":go::robot_face: Lint"
+    key: lint
+    command: .buildkite/steps/lint.sh
+    plugins:
+      - docker#v5.11.0:
+          image: "golang:1.22"
+
+  - name: ":go::test_tube: Test"
+    key: test
+    command: ".buildkite/steps/test.sh"
+    artifact_paths: junit-*.xml
+    plugins:
+      - docker#v5.11.0:
+          image: "golang:1.22"
+          propagate-environment: true
+      - artifacts#v1.9.0:
+          upload: "cover.{html,out}"
+
+  - label: ":writing_hand: Annotate with Test Failures"
+    key: annotate
+    depends_on: test
+    allow_dependency_failure: true
+    plugins:
+      - junit-annotate#v1.6.0:
+          artifacts: junit-*.xml

.buildkite/steps/lint.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -Eeufo pipefail
+
+echo --- :go: Checking go mod tidiness
+go mod tidy
+if ! git diff --no-ext-diff --exit-code; then
+  echo ^^^ +++
+  echo "The go.mod or go.sum files are out of sync with the source code"
+  echo "Please run \`go mod tidy\` locally, and commit the result."
+
+  exit 1
+fi
+
+echo --- :go: Checking go formatting
+gofmt -w .
+if ! git diff --no-ext-diff --exit-code; then
+  echo ^^^ +++
+  echo "Files have not been formatted with gofmt."
+  echo "Fix this by running \`go fmt ./...\` locally, and committing the result."
+
+  exit 1
+fi
+
+echo +++ Everything is clean and tidy! 🎉

.buildkite/steps/test.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+go install gotest.tools/[email protected]
+
+echo '+++ Running tests'
+gotestsum --junitfile "junit-${BUILDKITE_JOB_ID}.xml" -- -count=1 -coverprofile=cover.out -race "$@" ./...
+
+echo 'Producing coverage report'
+go tool cover -html cover.out -o cover.html

README.md

@@ -1,6 +1,7 @@
 Interpolate
 ===========
 
+[![Build status](https://badge.buildkite.com/3d9081230a965a20d7b68d66564644febdada7f968cc6f9c9c.svg)](https://buildkite.com/buildkite/interpolate)
 [![GoDoc](https://godoc.org/github.com/buildkite/interpolate?status.svg)](https://godoc.org/github.com/buildkite/interpolate)
 
 A golang library for parameter expansion (like `${BLAH}` or `$BLAH`) in strings from environment variables. An implementation of [POSIX Parameter Expansion](http://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_06_02), plus some other basic operations that you'd expect in a shell scripting environment [like bash](https://www.gnu.org/software/bash/manual/html_node/Shell-Parameter-Expansion.html).
@@ -53,7 +54,10 @@ func main() {
   <dd><strong>Use the substring of parameter after offset of given length.</strong> A negative offset must be separated from the colon with a space, and will select from the end of the string. If the offset is out of bounds, an empty string will be substituted. If the length is greater than the length then the entire string will be returned.</dd>
 
   <dt><code>${parameter:?<em>[word]</em>}</code></dt>
-  <dd>Indicate Error if Null or Unset. If parameter is unset or null, the expansion of word (or a message indicating it is unset if word is omitted) shall be returned as an error.</dd>
+  <dd><strong>Indicate Error if Null or Unset.</strong> If parameter is unset or null, the expansion of word (or a message indicating it is unset if word is omitted) shall be returned as an error.</dd>
+
+  <dt><code>$$parameter</code> or <code>\$parameter</code> or <code>$${expression}</code> or <code>\${expression}</code></dt>
+  <dd><strong>An escaped interpolation.</strong> Will not be interpolated, but will be unescaped by a call to <code>interpolate.Interpolate()</code></dd>
 </dl>
 
 ## License

go.mod

@@ -1 +1,3 @@
 module github.com/buildkite/interpolate
+
+go 1.22.3

interpolate.go

@@ -82,6 +82,19 @@ func (e UnsetValueExpansion) Expand(env Env) (string, error) {
 	return val, nil
 }
 
+// EscapedExpansion is an expansion that is delayed until later on (usually by a later process)
+type EscapedExpansion struct {
+	Identifier string
+}
+
+func (e EscapedExpansion) Identifiers() []string {
+	return []string{"$" + e.Identifier}
+}
+
+func (e EscapedExpansion) Expand(Env) (string, error) {
+	return "$" + e.Identifier, nil
+}
+
 // SubstringExpansion returns a substring (or slice) of the env
 type SubstringExpansion struct {
 	Identifier string

interpolate_test.go

@@ -24,6 +24,8 @@ func ExampleInterpolate() {
 }
 
 func TestBasicInterpolation(t *testing.T) {
+	t.Parallel()
+
 	environ := interpolate.NewMapEnv(map[string]string{
 		"TEST1": "A test",
 		"TEST2": "Another",
@@ -49,6 +51,8 @@ func TestBasicInterpolation(t *testing.T) {
 		{`${TEST4}`, "Only one level of $TEST3 interpolation"},
 	} {
 		t.Run(tc.Str, func(t *testing.T) {
+			t.Parallel()
+
 			result, err := interpolate.Interpolate(environ, tc.Str)
 			if err != nil {
 				t.Fatal(err)
@@ -61,6 +65,8 @@ func TestBasicInterpolation(t *testing.T) {
 }
 
 func TestNestedInterpolation(t *testing.T) {
+	t.Parallel()
+
 	environ := interpolate.NewMapEnv(map[string]string{
 		"TEST1": "A test",
 		"TEST2": "Another",
@@ -77,6 +83,8 @@ func TestNestedInterpolation(t *testing.T) {
 		{`${TEST5:-Some text ${TEST2:-$TEST1} with $TEST3}`, "Some text Another with Llamas"},
 	} {
 		t.Run(tc.Str, func(t *testing.T) {
+			t.Parallel()
+
 			result, err := interpolate.Interpolate(environ, tc.Str)
 			if err != nil {
 				t.Fatal(err)
@@ -89,6 +97,8 @@ func TestNestedInterpolation(t *testing.T) {
 }
 
 func TestIgnoresParentheses(t *testing.T) {
+	t.Parallel()
+
 	for _, str := range []string{
 		`$(echo hello world)`,
 		`testing $(echo hello world)`,
@@ -105,6 +115,8 @@ func TestIgnoresParentheses(t *testing.T) {
 }
 
 func TestVariablesMustStartWithLetters(t *testing.T) {
+	t.Parallel()
+
 	for _, str := range []string{
 		`$1 burgers`,
 		`$99bottles`,
@@ -119,6 +131,8 @@ func TestVariablesMustStartWithLetters(t *testing.T) {
 }
 
 func TestMissingParameterValuesReturnEmptyStrings(t *testing.T) {
+	t.Parallel()
+
 	for _, str := range []string{
 		`$BUILDKITE_COMMIT`,
 		`${BUILDKITE_COMMIT}`,
@@ -128,6 +142,8 @@ func TestMissingParameterValuesReturnEmptyStrings(t *testing.T) {
 		`${BUILDKITE_COMMIT:7:14}`,
 	} {
 		t.Run(str, func(t *testing.T) {
+			t.Parallel()
+
 			result, err := interpolate.Interpolate(nil, str)
 			if err != nil {
 				t.Fatal(err)
@@ -140,6 +156,8 @@ func TestMissingParameterValuesReturnEmptyStrings(t *testing.T) {
 }
 
 func TestSubstringsWithOffsets(t *testing.T) {
+	t.Parallel()
+
 	environ := interpolate.NewMapEnv(map[string]string{"BUILDKITE_COMMIT": "1adf998e39f647b4b25842f107c6ed9d30a3a7c7"})
 
 	for _, tc := range []struct {
@@ -171,6 +189,8 @@ func TestSubstringsWithOffsets(t *testing.T) {
 		{`${BUILDKITE_COMMIT:7:-128}`, ``},
 	} {
 		t.Run(tc.Str, func(t *testing.T) {
+			t.Parallel()
+
 			result, err := interpolate.Interpolate(environ, tc.Str)
 			if err != nil {
 				t.Fatal(err)
@@ -183,6 +203,8 @@ func TestSubstringsWithOffsets(t *testing.T) {
 }
 
 func TestInterpolateIsntGreedy(t *testing.T) {
+	t.Parallel()
+
 	environ := interpolate.NewMapEnv(map[string]string{
 		"BUILDKITE_COMMIT":       "cfeeee3fa7fa1a6311723f5cbff95b738ec6e683",
 		"BUILDKITE_PARALLEL_JOB": "456",
@@ -207,6 +229,8 @@ func TestInterpolateIsntGreedy(t *testing.T) {
 }
 
 func TestDefaultValues(t *testing.T) {
+	t.Parallel()
+
 	environ := interpolate.NewMapEnv(map[string]string{
 		"DAY":       "Blarghday",
 		"EMPTY_DAY": "",
@@ -236,6 +260,8 @@ func TestDefaultValues(t *testing.T) {
 }
 
 func TestRequiredVariables(t *testing.T) {
+	t.Parallel()
+
 	for _, tc := range []struct {
 		Str         string
 		ExpectedErr string
@@ -252,6 +278,8 @@ func TestRequiredVariables(t *testing.T) {
 }
 
 func TestEscapingVariables(t *testing.T) {
+	t.Parallel()
+
 	for _, tc := range []struct {
 		Str      string
 		Expected string
@@ -260,6 +288,8 @@ func TestEscapingVariables(t *testing.T) {
 		{`Do this \$ESCAPE_PARTY`, `Do this $ESCAPE_PARTY`},
 		{`Do this $${SUCH_ESCAPE}`, `Do this ${SUCH_ESCAPE}`},
 		{`Do this \${SUCH_ESCAPE}`, `Do this ${SUCH_ESCAPE}`},
+		{`Do this $${SUCH_ESCAPE:-$OTHERWISE}`, `Do this ${SUCH_ESCAPE:-$OTHERWISE}`},
+		{`Do this \${SUCH_ESCAPE:-$OTHERWISE}`, `Do this ${SUCH_ESCAPE:-$OTHERWISE}`},
 	} {
 		result, err := interpolate.Interpolate(nil, tc.Str)
 		if err != nil {
@@ -272,18 +302,22 @@ func TestEscapingVariables(t *testing.T) {
 }
 
 func TestExtractingIdentifiers(t *testing.T) {
+	t.Parallel()
+
 	for _, tc := range []struct {
 		Str         string
 		Identifiers []string
 	}{
 		{`Hello ${REQUIRED_VAR?}`, []string{`REQUIRED_VAR`}},
 		{`${LLAMAS:-${ROCK:-true}}`, []string{`LLAMAS`, `ROCK`}},
 		{`${BUILDKITE_COMMIT:0}`, []string{`BUILDKITE_COMMIT`}},
+		{`$BUILDKITE_COMMIT hello there $$DOUBLE_DOLLAR \$ESCAPED_DOLLAR`, []string{`BUILDKITE_COMMIT`, `$DOUBLE_DOLLAR`, `$ESCAPED_DOLLAR`}},
 	} {
 		id, err := interpolate.Identifiers(tc.Str)
 		if err != nil {
 			t.Fatal(err)
 		}
+
 		if !reflect.DeepEqual(id, tc.Identifiers) {
 			t.Fatalf("Test %q should have identifiers %v, got %v", tc.Str, tc.Identifiers, id)
 		}

parser.go

@@ -23,18 +23,20 @@ import (
 // and structs named the same reading the string bite by bite (peekRune and nextRune)
 
 /*
-EscapedBackslash = "\\"
-EscapedDollar    = ( "\$" | "$$")
-Identifier       = letter { letters | digit | "_" }
-Expansion        = "$" ( Identifier | Brace )
-Brace            = "{" Identifier [ Identifier BraceOperation ] "}"
-Text             = { EscapedBackslash | EscapedDollar | all characters except "$" }
-Expression       = { Text | Expansion }
-EmptyValue       = ":-" { Expression }
-UnsetValue       = "-" { Expression }
-Substring        = ":" number [ ":" number ]
-Required         = "?" { Expression }
-Operation        = EmptyValue | UnsetValue | Substring | Required
+EscapedBackslash   = "\\"
+Identifier         = letter { letters | digit | "_" }
+EscapedDollar      = ( "\$" | "$$" )
+EscapedExpansion   = EscapedDollar ( Identifier | Brace )
+UnescapedExpansion = "$" ( Identifier | Brace )
+Expansion          = UnescapedExpansion | EscapedExpansion
+Brace              = "{" Identifier [ Identifier BraceOperation ] "}"
+Text               = { EscapedBackslash | EscapedDollar | all characters except "$" }
+Expression         = { Text | Expansion }
+EmptyValue         = ":-" { Expression }
+UnsetValue         = "-" { Expression }
+Substring          = ":" number [ ":" number ]
+Required           = "?" { Expression }
+Operation          = EmptyValue | UnsetValue | Substring | Required
 */
 
 const (
@@ -75,9 +77,17 @@ func (p *Parser) parseExpression(stop ...rune) (Expression, error) {
 			p.pos += 2
 			expr = append(expr, ExpressionItem{Text: `\\`})
 			continue
-		} else if strings.HasPrefix(p.input[p.pos:], `\$`) || strings.HasPrefix(p.input[p.pos:], `$$`) {
+		}
+
+		if strings.HasPrefix(p.input[p.pos:], `\$`) || strings.HasPrefix(p.input[p.pos:], `$$`) {
 			p.pos += 2
-			expr = append(expr, ExpressionItem{Text: `$`})
+
+			ee, err := p.parseEscapedExpansion()
+			if err != nil {
+				return nil, err
+			}
+
+			expr = append(expr, ExpressionItem{Expansion: ee})
 			continue
 		}
 
@@ -112,6 +122,23 @@ func (p *Parser) parseExpression(stop ...rune) (Expression, error) {
 	return expr, nil
 }
 
+func (p *Parser) parseEscapedExpansion() (Expansion, error) {
+	// if it's an escaped brace expansion, (eg $${MY_COOL_VAR:-5}) consume text until the close brace
+	if c := p.peekRune(); c == '{' {
+		id := p.scanUntil(func(r rune) bool { return r == '}' })
+		id = id + string(p.nextRune()) // we know that the next rune is a close brace, chuck it on the end
+		return EscapedExpansion{Identifier: id}, nil
+	}
+
+	// otherwise, it's an escaped identifier (eg $$MY_COOL_VAR)
+	id, err := p.scanIdentifier()
+	if err != nil {
+		return nil, err
+	}
+
+	return EscapedExpansion{Identifier: id}, nil
+}
+
 func (p *Parser) parseExpansion() (Expansion, error) {
 	if c := p.nextRune(); c != '$' {
 		return nil, fmt.Errorf("Expected expansion to start with $, got %c", c)

parser_test.go

@@ -8,6 +8,8 @@ import (
 )
 
 func TestParser(t *testing.T) {
+	t.Parallel()
+
 	var testCases = []struct {
 		String   string
 		Expected []interpolate.ExpressionItem
@@ -73,17 +75,15 @@ func TestParser(t *testing.T) {
 		{
 			String: `\${HELLO_WORLD-blah}`,
 			Expected: []interpolate.ExpressionItem{
-				{Text: `$`},
-				{Text: `{HELLO_WORLD-blah}`},
+				{Expansion: interpolate.EscapedExpansion{Identifier: "{HELLO_WORLD-blah}"}},
 			},
 		},
 		{
 			String: `Test \\\${HELLO_WORLD-blah}`,
 			Expected: []interpolate.ExpressionItem{
 				{Text: `Test `},
 				{Text: `\\`},
-				{Text: `$`},
-				{Text: `{HELLO_WORLD-blah}`},
+				{Expansion: interpolate.EscapedExpansion{Identifier: "{HELLO_WORLD-blah}"}},
 			},
 		},
 		{
@@ -167,10 +167,16 @@ func TestParser(t *testing.T) {
 				{Text: `echo hello world)`},
 			},
 		},
+		{
+			String:   "$$MOUNTAIN",
+			Expected: []interpolate.ExpressionItem{{Expansion: interpolate.EscapedExpansion{Identifier: "MOUNTAIN"}}},
+		},
 	}
 
 	for _, tc := range testCases {
 		t.Run(tc.String, func(t *testing.T) {
+			t.Parallel()
+
 			actual, err := interpolate.NewParser(tc.String).Parse()
 			if err != nil {
 				t.Fatal(err)