Skip to content

Commit

Permalink
Merge pull request #218 from buildkite/tet-395/support-muted-test
Browse files Browse the repository at this point in the history 
Support muted test
  • Loading branch information
@nprizal
nprizal authored Nov 13, 2024
2 parents ff90e35 + fc20a43 commit 7301e6b
Show file tree
Hide file tree
Showing 20 changed files with 515 additions and 146 deletions.
115 changes: 113 additions & 2 deletions internal/api/create_test_plan_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import (
func TestCreateTestPlan(t *testing.T) {
mockProvider, err := consumer.NewV2Pact(consumer.MockHTTPProviderConfig{
Consumer: "TestEngineClient",
Provider: "TestPlanServer",
Provider: "TestEngineServer",
})

if err != nil {
Expand Down Expand Up @@ -122,7 +122,7 @@ func TestCreateTestPlan(t *testing.T) {
func TestCreateTestPlan_SplitByExample(t *testing.T) {
mockProvider, err := consumer.NewV2Pact(consumer.MockHTTPProviderConfig{
Consumer: "TestEngineClient",
Provider: "TestPlanServer",
Provider: "TestEngineServer",
})

if err != nil {
Expand All @@ -145,6 +145,7 @@ func TestCreateTestPlan_SplitByExample(t *testing.T) {
},
},
},
Runner: "rspec",
}

err = mockProvider.
Expand Down Expand Up @@ -274,6 +275,116 @@ func TestCreateTestPlan_BadRequest(t *testing.T) {
}
}

func TestCreateTestPlan_MutedTests(t *testing.T) {
mockProvider, err := consumer.NewV2Pact(consumer.MockHTTPProviderConfig{
Consumer: "TestEngineClient",
Provider: "TestEngineServer",
})

if err != nil {
t.Error("Error mocking provider", err)
}

params := TestPlanParams{
Runner: "rspec",
Branch: "tet-123-add-branch-name",
Identifier: "abc123",
Parallelism: 3,
Tests: TestPlanParamsTest{
Files: []plan.TestCase{
{Path: "sky_spec.rb"},
},
},
}

err = mockProvider.
AddInteraction().
Given("A test plan doesn't exist and muted test exists").
UponReceiving("A request to create test plan with identifier abc123 and split by example disabled").
WithRequest("POST", "/v2/analytics/organizations/buildkite/suites/rspec/test_plan", func(b *consumer.V2RequestBuilder) {
b.Header("Authorization", matchers.String("Bearer asdf1234"))
b.Header("Content-Type", matchers.String("application/json"))
b.JSONBody(params)
}).
WillRespondWith(200, func(b *consumer.V2ResponseBuilder) {
b.Header("Content-Type", matchers.String("application/json; charset=utf-8"))
b.JSONBody(matchers.MapMatcher{
"tasks": matchers.Like(map[string]interface{}{
"0": matchers.Like(map[string]interface{}{
"node_number": matchers.Like(0),
"tests": matchers.EachLike(matchers.MapMatcher{
"path": matchers.Like("sky_spec.rb"),
"format": matchers.Like("file"),
"estimated_duration": matchers.Like(1000),
}, 1),
}),
"1": matchers.Like(map[string]interface{}{
"node_number": matchers.Like(1),
"tests": []plan.TestCase{},
}),
"2": matchers.Like(map[string]interface{}{
"node_number": matchers.Like(2),
"tests": []plan.TestCase{},
}),
}),
"muted_tests": matchers.EachLike(matchers.MapMatcher{
"path": matchers.Like("./turtle_spec.rb:3"),
"scope": matchers.Like("turtle"),
"name": matchers.Like("is green"),
}, 1),
})
}).
ExecuteTest(t, func(config consumer.MockServerConfig) error {
ctx := context.Background()
fetchCtx, cancel := context.WithTimeout(ctx, 1*time.Second)
defer cancel()

url := fmt.Sprintf("http://%s:%d", config.Host, config.Port)
apiClient := NewClient(ClientConfig{
AccessToken: "asdf1234",
OrganizationSlug: "buildkite",
ServerBaseUrl: url,
})

got, err := apiClient.CreateTestPlan(fetchCtx, "rspec", params)
if err != nil {
t.Errorf("CreateTestPlan(ctx, %v) error = %v", params, err)
}

want := plan.TestPlan{
Tasks: map[string]*plan.Task{
"0": {
NodeNumber: 0,
Tests: []plan.TestCase{{
Path: "sky_spec.rb",
Format: "file",
EstimatedDuration: 1000,
}},
},
"1": {
NodeNumber: 1,
Tests: []plan.TestCase{},
},
"2": {
NodeNumber: 2,
Tests: []plan.TestCase{},
},
},
MutedTests: []plan.TestCase{{Name: "is green", Path: "./turtle_spec.rb:3", Scope: "turtle"}},
}

if diff := cmp.Diff(got, want); diff != "" {
t.Errorf("CreateTestPlan(ctx, %v) diff (-got +want):\n%s", params, diff)
}

return nil
})

if err != nil {
t.Error("mockProvider error", err)
}
}

func TestCreateTestPlan_InternalServerError(t *testing.T) {
originalTimeout := retryTimeout
retryTimeout = 1 * time.Millisecond
Expand Down
2 changes: 1 addition & 1 deletion internal/api/fetch_files_timing_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import (
func TestFetchFilesTiming(t *testing.T) {
mockProvider, err := consumer.NewV2Pact(consumer.MockHTTPProviderConfig{
Consumer: "TestEngineClient",
Provider: "TestPlanServer",
Provider: "TestEngineServer",
})

if err != nil {
Expand Down
4 changes: 2 additions & 2 deletions internal/api/fetch_test_plan_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import (
func TestFetchTestPlan(t *testing.T) {
mockProvider, err := consumer.NewV2Pact(consumer.MockHTTPProviderConfig{
Consumer: "TestEngineClient",
Provider: "TestPlanServer",
Provider: "TestEngineServer",
})

if err != nil {
Expand Down Expand Up @@ -99,7 +99,7 @@ func TestFetchTestPlan(t *testing.T) {
func TestFetchTestPlan_NotFound(t *testing.T) {
mockProvider, err := consumer.NewV2Pact(consumer.MockHTTPProviderConfig{
Consumer: "TestEngineClient",
Provider: "TestPlanServer",
Provider: "TestEngineServer",
})

if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion internal/api/filter_tests_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import (
func TestFilterTests_SlowFiles(t *testing.T) {
mockProvider, err := consumer.NewV2Pact(consumer.MockHTTPProviderConfig{
Consumer: "TestEngineClient",
Provider: "TestPlanServer",
Provider: "TestEngineServer",
})

if err != nil {
Expand Down
4 changes: 2 additions & 2 deletions internal/api/post_test_plan_metadata_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
func TestPostTestPlanMetadata(t *testing.T) {
mockProvider, err := consumer.NewV2Pact(consumer.MockHTTPProviderConfig{
Consumer: "TestEngineClient",
Provider: "TestPlanServer",
Provider: "TestEngineServer",
})

if err != nil {
Expand Down Expand Up @@ -85,7 +85,7 @@ func TestPostTestPlanMetadata(t *testing.T) {
func TestPostTestPlanMetadata_NotFound(t *testing.T) {
mockProvider, err := consumer.NewV2Pact(consumer.MockHTTPProviderConfig{
Consumer: "TestEngineClient",
Provider: "TestPlanServer",
Provider: "TestEngineServer",
})

if err != nil {
Expand Down
1 change: 1 addition & 0 deletions internal/plan/type.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,5 @@ type TestPlan struct {
Experiment string `json:"experiment"`
Tasks map[string]*Task `json:"tasks"`
Fallback bool
MutedTests []TestCase `json:"muted_tests,omitempty"`
}
8 changes: 6 additions & 2 deletions internal/runner/cypress.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,12 @@ func NewCypress(c RunnerConfig) Cypress {
return Cypress{c}
}

func (c Cypress) Run(testCases []string, retry bool) (RunResult, error) {
cmdName, cmdArgs, err := c.commandNameAndArgs(c.TestCommand, testCases)
func (c Cypress) Run(testCases []plan.TestCase, retry bool) (RunResult, error) {
testPaths := make([]string, len(testCases))
for i, tc := range testCases {
testPaths[i] = tc.Path
}
cmdName, cmdArgs, err := c.commandNameAndArgs(c.TestCommand, testPaths)
if err != nil {
return RunResult{Status: RunStatusError}, fmt.Errorf("failed to build command: %w", err)
}
Expand Down
42 changes: 25 additions & 17 deletions internal/runner/cypress_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"syscall"
"testing"

"github.com/buildkite/test-engine-client/internal/plan"
"github.com/google/go-cmp/cmp"
"github.com/kballard/go-shellquote"
)
Expand All @@ -17,19 +18,21 @@ func TestCypressRun(t *testing.T) {
TestCommand: "yarn cypress run --spec {{testExamples}}",
})

files := []string{"./cypress/e2e/passing_spec.cy.js"}
got, err := cypress.Run(files, false)
testCases := []plan.TestCase{
{Path: "./cypress/e2e/passing_spec.cy.js"},
}
got, err := cypress.Run(testCases, false)

want := RunResult{
Status: RunStatusPassed,
}

if err != nil {
t.Errorf("Cypress.Run(%q) error = %v", files, err)
t.Errorf("Cypress.Run(%q) error = %v", testCases, err)
}

if diff := cmp.Diff(got, want); diff != "" {
t.Errorf("Cypress.Run(%q) diff (-got +want):\n%s", files, diff)
t.Errorf("Cypress.Run(%q) diff (-got +want):\n%s", testCases, diff)
}
}

Expand All @@ -40,20 +43,23 @@ func TestCypressRun_TestFailed(t *testing.T) {
TestCommand: "yarn cypress run --spec {{testExamples}}",
})

files := []string{"./cypress/e2e/failing_spec.cy.js", "./cypress/e2e/passing_spec.cy.js"}
got, err := cypress.Run(files, false)
testCases := []plan.TestCase{
{Path: "./cypress/e2e/failing_spec.cy.js"},
{Path: "./cypress/e2e/passing_spec.cy.js"},
}
got, err := cypress.Run(testCases, false)

want := RunResult{
Status: RunStatusError,
}

if diff := cmp.Diff(got, want); diff != "" {
t.Errorf("Cypress.Run(%q) diff (-got +want):\n%s", files, diff)
t.Errorf("Cypress.Run(%q) diff (-got +want):\n%s", testCases, diff)
}

exitError := new(exec.ExitError)
if !errors.As(err, &exitError) {
t.Errorf("Cypress.Run(%q) error type = %T (%v), want *exec.ExitError", files, err, err)
t.Errorf("Cypress.Run(%q) error type = %T (%v), want *exec.ExitError", testCases, err, err)
}
}

Expand All @@ -64,45 +70,47 @@ func TestCypressRun_CommandFailed(t *testing.T) {
},
}

files := []string{}
got, err := cypress.Run(files, false)
testCases := []plan.TestCase{}
got, err := cypress.Run(testCases, false)

want := RunResult{
Status: RunStatusError,
}

if diff := cmp.Diff(got, want); diff != "" {
t.Errorf("Cypress.Run(%q) diff (-got +want):\n%s", files, diff)
t.Errorf("Cypress.Run(%q) diff (-got +want):\n%s", testCases, diff)
}

exitError := new(exec.ExitError)
if !errors.As(err, &exitError) {
t.Errorf("Cypress.Run(%q) error type = %T (%v), want *exec.ExitError", files, err, err)
t.Errorf("Cypress.Run(%q) error type = %T (%v), want *exec.ExitError", testCases, err, err)
}
}

func TestCypressRun_SignaledError(t *testing.T) {
cypress := NewCypress(RunnerConfig{
TestCommand: "./testdata/segv.sh",
})
files := []string{"./doesnt-matter.cy.js"}
testCases := []plan.TestCase{
{Path: "./doesnt-matter.cy.js"},
}

got, err := cypress.Run(files, false)
got, err := cypress.Run(testCases, false)

want := RunResult{
Status: RunStatusError,
}

if diff := cmp.Diff(got, want); diff != "" {
t.Errorf("Cypress.Run(%q) diff (-got +want):\n%s", files, diff)
t.Errorf("Cypress.Run(%q) diff (-got +want):\n%s", testCases, diff)
}

signalError := new(ProcessSignaledError)
if !errors.As(err, &signalError) {
t.Errorf("Cypress.Run(%q) error type = %T (%v), want *ErrProcessSignaled", files, err, err)
t.Errorf("Cypress.Run(%q) error type = %T (%v), want *ErrProcessSignaled", testCases, err, err)
}
if signalError.Signal != syscall.SIGSEGV {
t.Errorf("Cypress.Run(%q) signal = %d, want %d", files, syscall.SIGSEGV, signalError.Signal)
t.Errorf("Cypress.Run(%q) signal = %d, want %d", testCases, syscall.SIGSEGV, signalError.Signal)
}
}

Expand Down
2 changes: 1 addition & 1 deletion internal/runner/detector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ type RunnerConfig struct {
}

type TestRunner interface {
Run(testCases []string, retry bool) (RunResult, error)
Run(testCases []plan.TestCase, retry bool) (RunResult, error)
GetExamples(files []string) ([]plan.TestCase, error)
GetFiles() ([]string, error)
Name() string
Expand Down
Loading

0 comments on commit 7301e6b

Please sign in to comment.