Merge pull request #435 from lwan-wanglin/parallel-bk

re-enable run case in parallel

controllers/operator/clusterpodplacementconfig_controller_test.go

@@ -38,7 +38,7 @@ import (
 	"github.com/openshift/multiarch-tuning-operator/pkg/utils"
 )
 
-var _ = Describe("Controllers/ClusterPodPlacementConfig/ClusterPodPlacementConfigReconciler", Ordered, func() {
+var _ = Describe("Controllers/ClusterPodPlacementConfig/ClusterPodPlacementConfigReconciler", Serial, Ordered, func() {
 	When("The ClusterPodPlacementConfig", func() {
 		Context("is handling the lifecycle of the operand", func() {
 			BeforeEach(func() {

controllers/operator/suite_test.go

@@ -18,11 +18,13 @@ package operator
 
 import (
 	"context"
+	"encoding/json"
 	"net/http"
 	"path/filepath"
 	"testing"
 	"time"
 
+	"github.com/openshift/multiarch-tuning-operator/pkg/e2e"
 	"github.com/openshift/multiarch-tuning-operator/pkg/informers"
 
 	"sigs.k8s.io/kustomize/api/resmap"
@@ -33,13 +35,12 @@ import (
 	"sigs.k8s.io/kustomize/api/krusty"
 	"sigs.k8s.io/kustomize/kyaml/filesys"
 
-	admissionv1 "k8s.io/api/admissionregistration/v1"
-	appsv1 "k8s.io/api/apps/v1"
-	corev1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/tools/clientcmd/api"
 
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -56,10 +57,6 @@ import (
 
 	"github.com/openshift/library-go/pkg/operator/events"
 
-	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
-
-	"github.com/openshift/multiarch-tuning-operator/apis/multiarch/v1alpha1"
-	"github.com/openshift/multiarch-tuning-operator/apis/multiarch/v1beta1"
 	podplacement "github.com/openshift/multiarch-tuning-operator/controllers/podplacement"
 	testingutils "github.com/openshift/multiarch-tuning-operator/pkg/testing/framework"
 	"github.com/openshift/multiarch-tuning-operator/pkg/utils"
@@ -78,14 +75,18 @@ var (
 	suiteLog  = ctrl.Log.WithName("setup")
 )
 
+func init() {
+	e2e.CommonInit()
+}
+
 func TestOperator(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "Operator Integration Suite", Label("integration", "operator"))
 }
 
 var _ = BeforeAll
 
-var _ = BeforeSuite(func() {
+var _ = SynchronizedBeforeSuite(func() []byte {
 	suiteLog = zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true), zap.Level(zapcore.Level(-5)))
 	ctx = context.TODO()
 	logf.SetLogger(suiteLog)
@@ -94,9 +95,32 @@ var _ = BeforeSuite(func() {
 	startTestEnv()
 	testingutils.EnsureNamespaces(ctx, k8sClient, "test-namespace")
 	runManager()
+	kc := testingutils.FromEnvTestConfig(cfg)
+	data, err := json.Marshal(kc)
+	Expect(err).NotTo(HaveOccurred(), "failed to marshal sharedData")
+	return data
+}, func(data []byte) {
+	suiteLog = zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true), zap.Level(zapcore.Level(-5)))
+	ctx = context.TODO()
+	logf.SetLogger(suiteLog)
+	SetDefaultEventuallyPollingInterval(5 * time.Millisecond)
+	SetDefaultEventuallyTimeout(5 * time.Second)
+	var err error
+	var kc api.Config
+	err = json.Unmarshal(data, &kc)
+	Expect(err).NotTo(HaveOccurred(), "failed to unmarshal sharedData")
+	// Sync test cluster environment
+	ocg := clientcmd.NewDefaultClientConfig(kc, &clientcmd.ConfigOverrides{})
+	cfg, err = ocg.ClientConfig()
+	Expect(err).NotTo(HaveOccurred())
+	Expect(cfg).NotTo(BeNil())
+
+	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
+	Expect(err).NotTo(HaveOccurred())
+	Expect(k8sClient).NotTo(BeNil())
 })
 
-var _ = AfterSuite(func() {
+var _ = SynchronizedAfterSuite(func() {}, func() {
 	By("tearing down the test environment")
 	stopMgr()
 	// wait for the manager to stop. FIXME: this is a hack, not sure what is the right way to do it.
@@ -112,23 +136,10 @@ func startTestEnv() {
 		ErrorIfCRDPathMissing: true,
 	}
 	var err error
-
 	// cfg is defined in this file globally.
 	cfg, err = testEnv.Start()
 	Expect(err).NotTo(HaveOccurred())
 	Expect(cfg).NotTo(BeNil())
-	err = corev1.AddToScheme(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
-	err = appsv1.AddToScheme(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
-	err = v1alpha1.AddToScheme(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
-	err = v1beta1.AddToScheme(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
-	err = admissionv1.AddToScheme(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
-	err = monitoringv1.AddToScheme(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
 
 	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
 	Expect(err).NotTo(HaveOccurred())

controllers/podplacement/scheduling_gate_mutating_webhook.go

@@ -18,6 +18,7 @@ package podplacement
 
 import (
 	"context"
+	"sync"
 	"time"
 
 	"net/http"
@@ -48,6 +49,7 @@ type PodSchedulingGateMutatingWebHook struct {
 	client     client.Client
 	clientSet  *kubernetes.Clientset
 	decoder    admission.Decoder
+	once       sync.Once
 	scheme     *runtime.Scheme
 	recorder   record.EventRecorder
 	workerPool *ants.MultiPool
@@ -65,9 +67,9 @@ func (a *PodSchedulingGateMutatingWebHook) Handle(ctx context.Context, req admis
 	responseTimeStart := time.Now()
 	defer utils.HistogramObserve(responseTimeStart, metrics.ResponseTime)
 	metrics.ProcessedPodsWH.Inc()
-	if a.decoder == nil {
+	a.once.Do(func() {
 		a.decoder = admission.NewDecoder(a.scheme)
-	}
+	})
 	pod := &Pod{
 		ctx:      ctx,
 		recorder: nil, // do we want to publish events if the pod is ignored?

controllers/podplacement/suite_test.go

@@ -42,9 +42,12 @@ import (
 	v1 "k8s.io/api/admissionregistration/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/json"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/tools/clientcmd/api"
 
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -56,6 +59,8 @@ import (
 
 	"github.com/panjf2000/ants/v2"
 
+	"github.com/openshift/multiarch-tuning-operator/pkg/e2e"
+	"github.com/openshift/multiarch-tuning-operator/pkg/testing/framework"
 	testingutils "github.com/openshift/multiarch-tuning-operator/pkg/testing/framework"
 	"github.com/openshift/multiarch-tuning-operator/pkg/testing/image/fake/registry"
 	"github.com/openshift/multiarch-tuning-operator/pkg/utils"
@@ -77,20 +82,32 @@ var (
 	suiteLog = ctrl.Log.WithName("setup")
 )
 
+type sharedData struct {
+	Kubeconfig       api.Config `json:"kubeconfig"`
+	RegistryAddress  string     `json:"registryAddress"`
+	RegistryCert     []byte     `json:"registryCert"`
+	RegistryCertPath string     `json:"registryCertpath"`
+}
+
+func init() {
+	e2e.CommonInit()
+}
+
 func TestAPIs(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "Controllers Integration Suite", Label("integration"))
 }
 
 var _ = BeforeAll
 
-var _ = BeforeSuite(func() {
+var _ = SynchronizedBeforeSuite(func() []byte {
 	suiteLog = zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true), zap.Level(zapcore.Level(-5)))
 
 	logf.SetLogger(suiteLog)
 	SetDefaultEventuallyPollingInterval(5 * time.Millisecond)
 	SetDefaultEventuallyTimeout(5 * time.Second)
 
+	By("Set up a shared test environment and local registry for testing on process 1 node")
 	wg := &sync.WaitGroup{}
 	testingutils.DecorateWithWaitGroup(wg, startTestEnv)
 	testingutils.DecorateWithWaitGroup(wg, startRegistry)
@@ -114,9 +131,50 @@ var _ = BeforeSuite(func() {
 
 	// TODO: should we continue running the manager in the BeforeSuite node?
 	runManager()
+
+	By("Prepare shared data and Pass to all processes")
+	// Get cluster info and share with all processes
+	kc := framework.FromEnvTestConfig(cfg)
+	// The registry.perRegistryCertDirPath is used by registry.PushMockImage,
+	// need to pass to all processes
+	registryCertPath := registry.GetCertPath()
+	data := sharedData{
+		Kubeconfig:       kc,
+		RegistryAddress:  registryAddress,
+		RegistryCert:     registryCert,
+		RegistryCertPath: registryCertPath,
+	}
+	jsonData, err := json.Marshal(data)
+	Expect(err).NotTo(HaveOccurred(), "failed to marshal sharedData")
+	return jsonData
+}, func(data []byte) {
+	suiteLog = zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true), zap.Level(zapcore.Level(-5)))
+
+	logf.SetLogger(suiteLog)
+	SetDefaultEventuallyPollingInterval(5 * time.Millisecond)
+	SetDefaultEventuallyTimeout(5 * time.Second)
+	var err error
+	By("Get shared data to all processes")
+	var sharedData sharedData
+	err = json.Unmarshal(data, &sharedData)
+	Expect(err).NotTo(HaveOccurred(), "failed to unmarshal sharedData")
+	// Sync registry.perRegistryCertDirPath for registry.PushMockImage
+	registryCertPath := sharedData.RegistryCertPath
+	registry.SetCertPath(registryCertPath)
+	// Sync registryAddress and registryCert
+	registryAddress = sharedData.RegistryAddress
+	registryCert = sharedData.RegistryCert
+	// Sync test cluster environment
+	kc := sharedData.Kubeconfig
+	ocg := clientcmd.NewDefaultClientConfig(kc, &clientcmd.ConfigOverrides{})
+	cfg, err = ocg.ClientConfig()
+	Expect(err).NotTo(HaveOccurred(), "Error loading kubeconfig")
+	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
+	Expect(err).NotTo(HaveOccurred())
+	Expect(k8sClient).NotTo(BeNil())
 })
 
-var _ = AfterSuite(func() {
+var _ = SynchronizedAfterSuite(func() {}, func() {
 	By("tearing down the test environment")
 	stopMgr()
 	// TODO: we miss a way to gracefully shutdown the registry server in the AfterSuite fixture.
@@ -142,9 +200,6 @@ func startTestEnv() {
 	Expect(err).NotTo(HaveOccurred())
 	Expect(cfg).NotTo(BeNil())
 
-	err = corev1.AddToScheme(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
-
 	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
 	Expect(err).NotTo(HaveOccurred())
 	Expect(k8sClient).NotTo(BeNil())

hack/ci-test.sh

@@ -23,7 +23,7 @@ REPO_ROOT=$(dirname "${BASH_SOURCE}")/..
 OPENSHIFT_CI=${OPENSHIFT_CI:-""}
 export ARTIFACT_DIR=${ARTIFACT_DIR:-$(mktemp -d)}
 GINKGO=${GINKGO:-"go run ${REPO_ROOT}/vendor/github.com/onsi/ginkgo/v2/ginkgo"}
-GINKGO_ARGS=${GINKGO_ARGS:-"-vv --randomize-all --randomize-suites -race -trace --keep-going --timeout=40m "}
+GINKGO_ARGS=${GINKGO_ARGS:-"-vv --randomize-all --randomize-suites -p -race -trace --keep-going --timeout=40m "}
 TEST_LABEL=${TEST_LABEL:-"integration"}
 GINKGO_ARGS="${GINKGO_ARGS} --label-filter ${TEST_LABEL}"
 GINKGO_EXTRA_ARGS=${GINKGO_EXTRA_ARGS:-""}

pkg/e2e/operator/pod_placement_config_test.go

@@ -23,7 +23,7 @@ const (
 	helloOpenshiftPublicMultiarchImage = "quay.io/openshifttest/hello-openshift:1.2.0"
 )
 
-var _ = Describe("The Multiarch Tuning Operator", func() {
+var _ = Describe("The Multiarch Tuning Operator", Serial, func() {
 	var (
 		podLabel                  = map[string]string{"app": "test"}
 		schedulingGateLabel       = map[string]string{utils.SchedulingGateLabel: utils.SchedulingGateLabelValueRemoved}

pkg/e2e/podplacement/e2e_test.go

@@ -14,15 +14,11 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/kubernetes/scheme"
 
 	ctrl "sigs.k8s.io/controller-runtime"
 	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
 
-	ocpappsv1 "github.com/openshift/api/apps/v1"
-	ocpbuildv1 "github.com/openshift/api/build/v1"
 	ocpconfigv1 "github.com/openshift/api/config/v1"
-	ocpmachineconfigurationv1 "github.com/openshift/api/machineconfiguration/v1"
 	ocpoperatorv1alpha1 "github.com/openshift/api/operator/v1alpha1"
 
 	"github.com/openshift/multiarch-tuning-operator/apis/multiarch/v1beta1"
@@ -58,23 +54,9 @@ func TestE2E(t *testing.T) {
 	RunSpecs(t, "Multiarch Tuning Operator Suite (PodPlacementOperand E2E)", Label("e2e", "pod-placement-operand"))
 }
 
-var _ = BeforeSuite(func() {
+var _ = SynchronizedBeforeSuite(func() []byte {
+	var err error
 	client, clientset, ctx, suiteLog = e2e.CommonBeforeSuite()
-	err := ocpappsv1.Install(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
-
-	err = ocpbuildv1.Install(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
-
-	err = ocpconfigv1.Install(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
-
-	err = ocpmachineconfigurationv1.Install(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
-
-	err = ocpoperatorv1alpha1.Install(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
-
 	err = client.Create(ctx, &v1beta1.ClusterPodPlacementConfig{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "cluster",
@@ -93,9 +75,16 @@ var _ = BeforeSuite(func() {
 		By("Wait for machineconfig finishing updating")
 		framework.WaitForMCPComplete(ctx, client)
 	}
+	return nil
+}, func(data []byte) {
+	var err error
+	client, clientset, ctx, suiteLog = e2e.CommonBeforeSuite()
+
+	masterNodes, err = framework.GetNodesWithLabel(ctx, client, "node-role.kubernetes.io/master", "")
+	Expect(err).NotTo(HaveOccurred())
 })
 
-var _ = AfterSuite(func() {
+var _ = SynchronizedAfterSuite(func() {}, func() {
 	err := client.Delete(ctx, &v1beta1.ClusterPodPlacementConfig{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "cluster",

pkg/e2e/suites_lifecycle.go

@@ -29,15 +29,15 @@ func CommonInit() {
 func CommonBeforeSuite() (client runtimeclient.Client, clientset *kubernetes.Clientset,
 	ctx context.Context, suiteLog logr.Logger) {
 	var err error
+	suiteLog = zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true), zap.Level(zapcore.Level(-5)))
+	logf.SetLogger(suiteLog)
+
 	client, err = framework.LoadClient()
 	Expect(err).ToNot(HaveOccurred())
 
 	clientset, err = framework.LoadClientset()
 	Expect(err).ToNot(HaveOccurred())
 
-	suiteLog = zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true), zap.Level(zapcore.Level(-5)))
-	logf.SetLogger(suiteLog)
-
 	ctx = context.Background()
 
 	SetDefaultEventuallyPollingInterval(PollingInterval)