WIP OPRUN-3962: Add OTE for origin tests

openshift/extended-tests/Makefile

@@ -0,0 +1,81 @@
+# Get the directory where this Makefile is, so we can use it below for including
+# Include the same Bingo variables used by the project
+DIR := $(strip $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))))
+include $(DIR)/../../.bingo/Variables.mk
+
+# Definitions for the extended tests
+
+GO_PKG_NAME := github/operator-framework-operator-controller/openshift/extended-tests
+
+GIT_COMMIT := $(shell git rev-parse --short HEAD)
+BUILD_DATE := $(shell date -u +'%Y-%m-%dT%H:%M:%SZ')
+GIT_TREE_STATE := $(shell if git diff --quiet; then echo clean; else echo dirty; fi)
+
+LDFLAGS := -X '$(GO_PKG_NAME)/pkg/version.CommitFromGit=$(GIT_COMMIT)' \
+           -X '$(GO_PKG_NAME)/pkg/version.BuildDate=$(BUILD_DATE)' \
+           -X '$(GO_PKG_NAME)/pkg/version.GitTreeState=$(GIT_TREE_STATE)'
+
+#SECTION General
+
+# The help target prints out all targets with their descriptions organized
+# beneath their categories. The categories are represented by '#SECTION' and the
+# target descriptions by '#HELP' or '#EXHELP'. The awk commands is responsible for reading the
+# entire set of makefiles included in this invocation, looking for lines of the
+# file as xyz: #HELP something, and then pretty-format the target and help. Then,
+# if there's a line with #SECTION something, that gets pretty-printed as a category.
+# More info on the usage of ANSI control characters for terminal formatting:
+# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
+# More info on the awk command:
+# http://linuxcommand.org/lc3_adv_awk.php
+# The extended-help target uses '#EXHELP' as the delineator.
+
+.PHONY: help
+help: #HELP Display essential help.
+	@awk 'BEGIN {FS = ":[^#]*#HELP"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n\n"} /^[a-zA-Z_0-9-]+:.*#HELP / { printf "  \033[36m%-17s\033[0m %s\n", $$1, $$2 } ' $(MAKEFILE_LIST)
+
+#SECTION Tests
+TOOLS_BIN_DIR := $(CURDIR)/bin
+GINKGO := $(TOOLS_BIN_DIR)/ginkgo
+
+.PHONY: install-tools
+install-tools: $(GINKGO) #HELP Build vendored CLI tools
+
+$(GINKGO): vendor/modules.txt
+	go build -mod=vendor -o $(GINKGO) ./vendor/github.com/onsi/ginkgo/v2/ginkgo
+
+.PHONY: test-extended
+test-extended: install-tools $(GINKGO) #HELP Run the extended tests
+	$(GINKGO) $(if $(ARTIFACT_DIR),--output-dir='$(ARTIFACT_DIR)') \
+		--junit-report=junit_olm.xml ./test/...
+
+#SECTION Development
+.PHONY: verify #HELP To verify the code
+verify: tidy fmt vet lint
+
+.PHONY: tidy #HELP Run go mod tidy.
+tidy:
+	go mod tidy
+
+.PHONY: fmt
+fmt: #HELP Run go fmt against code.
+	go fmt ./...
+
+.PHONY: vet
+vet: #HELP Run go vet against code.
+	go vet ./...
+
+.PHONY: lint
+lint: $(GOLANGCI_LINT) #HELP Run golangci linter.
+	$(GOLANGCI_LINT) run
+
+.PHONY: fix-lint
+fix-lint: $(GOLANGCI_LINT) #HELP Fix lint issues
+	$(GOLANGCI_LINT) run --fix
+
+.PHONY: build
+build: #HELP Build the extended tests binary
+	# GO_COMPLIANCE_POLICY="exempt_all" must only be used for test related binaries.
+	# It prevents various FIPS compliance policies from being applied to this compilation.
+	# Do not set globally.
+	@mkdir -p $(TOOLS_BIN_DIR)
+	GO_COMPLIANCE_POLICY="exempt_all" go build -ldflags "$(LDFLAGS)" -o $(TOOLS_BIN_DIR)/extended-tests ./cmd/...

openshift/extended-tests/cmd/main.go

@@ -0,0 +1,129 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/openshift-eng/openshift-tests-extension/pkg/cmd"
+	e "github.com/openshift-eng/openshift-tests-extension/pkg/extension"
+	et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
+	g "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo"
+	// If using ginkgo, import your tests here
+	_ "github.com/openshift-eng/openshift-tests-extension/test/example"
+	"github.com/spf13/cobra"
+)
+
+func main() {
+	registry := e.NewRegistry()
+	ext := e.NewExtension("openshift", "payload", "olmv1-origin-tests-extension")
+
+	ext.AddSuite(
+		e.Suite{
+			Name:    "example/tests",
+			Parents: []string{"openshift/conformance/parallel"},
+		})
+
+	// The tests that a suite is composed of can be filtered by CEL expressions. By
+	// default, the qualifiers only apply to tests from this extension.
+	ext.AddSuite(e.Suite{
+		Name: "example/fast",
+		Qualifiers: []string{
+			`!labels.exists(l, l=="SLOW")`,
+		},
+	})
+
+	// Global suites' qualifiers will apply to all tests available, even
+	// those outside of this extension (when invoked by origin).
+	ext.AddGlobalSuite(e.Suite{
+		Name: "example/slow",
+		Qualifiers: []string{
+			`labels.exists(l, l=="SLOW")`,
+		},
+	})
+
+	// If using Ginkgo, build test specs automatically
+	specs, err := g.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite()
+	if err != nil {
+		panic(fmt.Sprintf("couldn't build extension test specs from ginkgo: %+v", err.Error()))
+	}
+
+	// Environment selector information can be added to test specs to support filtering by environment
+	specs.Select(et.NameContains("[sig-testing] openshift-tests-extension should support test-skips via environment flags")).
+		Include(et.PlatformEquals("aws"))
+
+	// You can add hooks to run before/after tests. There are BeforeEach, BeforeAll, AfterEach,
+	// and AfterAll. "Each" functions must be thread safe.
+	//
+	// specs.AddBeforeAll(func() {
+	// 	initializeTestFramework()
+	// })
+	//
+	// specs.AddBeforeEach(func(spec ExtensionTestSpec) {
+	//	if spec.Name == "my test" {
+	//		// do stuff
+	//	}
+	// })
+	//
+	// specs.AddAfterEach(func(res *ExtensionTestResult) {
+	// 	if res.Result == ResultFailed && apiTimeoutRegexp.Matches(res.Output) {
+	// 		res.AddDetails("api-timeout", collectDiagnosticInfo())
+	// 	}
+	// })
+
+	// You can also manually build a test specs list from other testing tooling
+	// TODO: example
+
+	// Modify specs, such as adding a label to all specs
+	// 	specs = specs.AddLabel("SLOW")
+
+	// Specs can be globally filtered...
+	// specs = specs.MustFilter([]string{`name.contains("filter")`})
+
+	// Or walked...
+	// specs = specs.Walk(func(spec *extensiontests.ExtensionTestSpec) {
+	//	if strings.Contains(e.Name, "scale up") {
+	//		e.Labels.Insert("SLOW")
+	//	}
+	//
+	// Specs can also be selected...
+	// specs = specs.Select(et.NameContains("slow test")).AddLabel("SLOW")
+	//
+	// Or with "any" (or) matching selections
+	// specs = specs.SelectAny(et.NameContains("slow test"), et.HasLabel("SLOW"))
+	//
+	// Or with "all" (and) matching selections
+	// specs = specs.SelectAll(et.NameContains("slow test"), et.HasTagWithValue("speed", "slow"))
+	//
+	// There are also Must* functions for any of the above flavors of selection
+	// which will return an error if nothing is found
+	// specs, err = specs.MustSelect(et.NameContains("slow test")).AddLabel("SLOW")
+	// if err != nil {
+	//    logrus.Warn("no specs found: %w", err)
+	// }
+	// Test renames
+	//	if spec.Name == "[sig-testing] openshift-tests-extension has a test with a typo" {
+	//		spec.OriginalName = `[sig-testing] openshift-tests-extension has a test with a tpyo`
+	//	}
+	//
+	// Filter by environment flags
+	// if spec.Name == "[sig-testing] openshift-tests-extension should support defining the platform for tests" {
+	//		spec.Include(et.PlatformEquals("aws"))
+	//		spec.Exclude(et.And(et.NetworkEquals("ovn"), et.TopologyEquals("ha")))
+	//	}
+	// })
+
+	ext.AddSpecs(specs)
+	registry.Register(ext)
+
+	root := &cobra.Command{
+		Long: "OLMv1 Origin Tests Extension",
+	}
+
+	root.AddCommand(cmd.DefaultExtensionCommands(registry)...)
+
+	if err := func() error {
+		return root.Execute()
+	}(); err != nil {
+		os.Exit(1)
+	}
+}

openshift/extended-tests/go.mod

@@ -0,0 +1,42 @@
+module github/operator-framework-operator-controller/openshift/extended-tests
+
+go 1.23.6
+
+require (
+	github.com/openshift-eng/openshift-tests-extension v0.0.0-20250522124649-4ffcd156ec7c
+	github.com/spf13/cobra v1.8.1
+)
+
+require (
+	cel.dev/expr v0.18.0 // indirect
+	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
+	github.com/google/cel-go v0.17.8 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/onsi/ginkgo/v2 v2.21.0 // indirect
+	github.com/onsi/gomega v1.35.1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stoewer/go-strcase v1.2.0 // indirect
+	golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
+	golang.org/x/net v0.30.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
+	golang.org/x/tools v0.26.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240924160255-9d4c2d233b61 // indirect
+	google.golang.org/protobuf v1.35.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)
+
+// cel-go v0.23.0 upgrade causes errors raised from the vendor source which lead to think in
+// incompatibilities scenarios. After upgrade to use the latest versions of k8s/api v0.33+
+// we should try to see if we could fix this one and remove this replace
+replace github.com/google/cel-go => github.com/google/cel-go v0.22.1
+
+replace k8s.io/api => k8s.io/api v0.32.3
+
+replace github.com/onsi/ginkgo/v2 => github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20241205171354-8006f302fd12

openshift/extended-tests/go.sum

@@ -0,0 +1,63 @@
+cel.dev/expr v0.18.0 h1:CJ6drgk+Hf96lkLikr4rFf19WrU0BOWEihyZnI2TAzo=
+cel.dev/expr v0.18.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw=
+github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
+github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
+github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
+github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
+github.com/google/cel-go v0.22.1 h1:AfVXx3chM2qwoSbM7Da8g8hX8OVSkBFwX+rz2+PcK40=
+github.com/google/cel-go v0.22.1/go.mod h1:BuznPXXfQDpXKWQ9sPW3TzlAJN5zzFe+i9tIs0yC4s8=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo=
+github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4=
+github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog=
+github.com/openshift-eng/openshift-tests-extension v0.0.0-20250522124649-4ffcd156ec7c h1:R5dI2oOF2RtS1sKtLrhW9KMg0ydzF0XM2Q//ma55nWI=
+github.com/openshift-eng/openshift-tests-extension v0.0.0-20250522124649-4ffcd156ec7c/go.mod h1:6gkP5f2HL0meusT0Aim8icAspcD1cG055xxBZ9yC68M=
+github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20241205171354-8006f302fd12 h1:AKx/w1qpS8We43bsRgf8Nll3CGlDHpr/WAXvuedTNZI=
+github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20241205171354-8006f302fd12/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
+github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU=
+github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU=
+golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
+golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
+golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
+golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
+golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ=
+golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0=
+google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 h1:YcyjlL1PRr2Q17/I0dPk2JmYS5CDXfcdb2Z3YRioEbw=
+google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240924160255-9d4c2d233b61 h1:N9BgCIAUvn/M+p4NJccWPWb3BWh88+zyL0ll9HgbEeM=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240924160255-9d4c2d233b61/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
+google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA=
+google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=