Add e2e tests

.github/workflows/go.yml

@@ -29,4 +29,4 @@ jobs:
       run: go build -mod=vendor -v ./...
 
     - name: Test
-      run: go test -race -mod=vendor -v ./...
+      run: go test -race -mod=vendor -v -timeout=30m ./...

Makefile

@@ -40,7 +40,7 @@ clean:
 .PHONY: clean
 
 test:
-	go test ./...
+	go test -timeout 30m ./...
 .PHONY: test
 
 default:

pkg/crd/crd_test.go

@@ -16,11 +16,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"github.com/testcontainers/testcontainers-go"
 	"github.com/testcontainers/testcontainers-go/modules/k3s"
-	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 type TestStore struct {
@@ -48,10 +44,6 @@ func (s *TestStore) Count() int32 {
 }
 
 func TestCRD(t *testing.T) {
-	scheme := clientgoscheme.Scheme
-	require.NoError(t, clientgoscheme.AddToScheme(scheme))
-	require.NoError(t, v1.AddToScheme(scheme))
-
 	crdPath := filepath.Join(t.TempDir(), "crd.yaml")
 	require.NoError(t, testutils.CopyFile("../../kustomize/bases/functions_crd.yaml", crdPath))
 	fnCrdPath := filepath.Join(t.TempDir(), "fn-crd.yaml")
@@ -65,13 +57,7 @@ func TestCRD(t *testing.T) {
 	require.NoError(t, k3sContainer.CopyFileToContainer(ctx, crdPath, filepath.Join(testutils.K3sManifests, "crd.yaml"), 0644))
 	require.NoError(t, k3sContainer.CopyFileToContainer(ctx, fnCrdPath, filepath.Join(testutils.K3sManifests, "fn-crd.yaml"), 0644))
 
-	kubeconfig, err := testutils.WriteKubeConfig(ctx, k3sContainer, t.TempDir())
-	require.NoError(t, err)
-
-	config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
-	require.NoError(t, err)
-	config.WarningHandler = rest.NoWarnings{}
-	ctrlCli, err := ctrlclient.New(config, ctrlclient.Options{})
+	_, ctrlCli, err := testutils.GetKubeConfig(ctx, k3sContainer)
 	require.NoError(t, err)
 
 	ts := &TestStore{t: t}

pkg/testutils/collector/collector.go

@@ -4,9 +4,10 @@ package collector
 
 import (
 	"context"
-	"errors"
 	"fmt"
+	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/Azure/adx-mon/pkg/testutils"
 	"github.com/testcontainers/testcontainers-go"
@@ -23,15 +24,22 @@ type CollectorContainer struct {
 }
 
 func Run(ctx context.Context, opts ...testcontainers.ContainerCustomizer) (*CollectorContainer, error) {
+	var relative string
+	for iter := range 4 {
+		relative = strings.Repeat("../", iter)
+		if _, err := os.Stat(filepath.Join(relative, "build/images/Dockerfile.ingestor")); err == nil {
+			break
+		}
+	}
+
 	req := testcontainers.ContainerRequest{
 		Name: "collector" + testcontainers.SessionID(),
 		FromDockerfile: testcontainers.FromDockerfile{
-			Repo:          DefaultImage,
-			Tag:           DefaultTag,
-			Context:       "../../..", // repo base
-			Dockerfile:    "build/images/Dockerfile.collector",
-			PrintBuildLog: true,
-			KeepImage:     true,
+			Repo:       DefaultImage,
+			Tag:        DefaultTag,
+			Context:    relative, // repo base
+			Dockerfile: "build/images/Dockerfile.collector",
+			KeepImage:  true,
 		},
 	}
 
@@ -98,12 +106,20 @@ func WithCluster(ctx context.Context, k *k3s.K3sContainer) testcontainers.Custom
 	}
 }
 
-// TODO Accept a collector configuration and write it to a temp directory, then transfer it to the cluster,
-// overwriting the existing configuration. Also create a default configuration if none is provided.
-// Note, this would require moving the configuration to somewhere it can be imported, get feedback
-// from the team.
-func WithConfig(ctx context.Context) testcontainers.CustomizeRequestOption {
-	return func(req *testcontainers.GenericContainerRequest) error {
-		return errors.New("not implemented")
+type KustoTableSchema struct{}
+
+func (k *KustoTableSchema) TableName() string {
+	return "Collector"
+}
+
+func (k *KustoTableSchema) CslColumns() []string {
+	return []string{
+		"msg:string",
+		"lvl:string",
+		"ts:datetime",
+		"namespace:string",
+		"container:string",
+		"pod:string",
+		"host:string",
 	}
 }

pkg/testutils/collector/collector_test.go

@@ -13,7 +13,7 @@ import (
 )
 
 func TestCollector(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
 	defer cancel()
 
 	c, err := collector.Run(ctx)

pkg/testutils/ingestor/ingestor.go

@@ -5,7 +5,9 @@ package ingestor
 import (
 	"context"
 	"fmt"
+	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/Azure/adx-mon/pkg/testutils"
 	"github.com/testcontainers/testcontainers-go"
@@ -22,15 +24,22 @@ type IngestorContainer struct {
 }
 
 func Run(ctx context.Context, opts ...testcontainers.ContainerCustomizer) (*IngestorContainer, error) {
+	var relative string
+	for iter := range 4 {
+		relative = strings.Repeat("../", iter)
+		if _, err := os.Stat(filepath.Join(relative, "build/images/Dockerfile.ingestor")); err == nil {
+			break
+		}
+	}
+
 	req := testcontainers.ContainerRequest{
 		Name: "ingestor" + testcontainers.SessionID(),
 		FromDockerfile: testcontainers.FromDockerfile{
-			Repo:          DefaultImage,
-			Tag:           DefaultTag,
-			Context:       "../../..", // repo base
-			Dockerfile:    "build/images/Dockerfile.ingestor",
-			PrintBuildLog: true,
-			KeepImage:     true,
+			Repo:       DefaultImage,
+			Tag:        DefaultTag,
+			Context:    relative, // repo base
+			Dockerfile: "build/images/Dockerfile.ingestor",
+			KeepImage:  true,
 		},
 	}
 
@@ -96,3 +105,21 @@ func WithCluster(ctx context.Context, k *k3s.K3sContainer) testcontainers.Custom
 		return nil
 	}
 }
+
+type KustoTableSchema struct{}
+
+func (k *KustoTableSchema) TableName() string {
+	return "Collector"
+}
+
+func (k *KustoTableSchema) CslColumns() []string {
+	return []string{
+		"msg:string",
+		"lvl:string",
+		"ts:datetime",
+		"namespace:string",
+		"container:string",
+		"pod:string",
+		"host:string",
+	}
+}

pkg/testutils/ingestor/ingestor_test.go

@@ -13,7 +13,7 @@ import (
 )
 
 func TestIngestor(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
 	defer cancel()
 
 	c, err := ingestor.Run(ctx)

pkg/testutils/ingestor/k8s.yaml

@@ -115,7 +115,6 @@ spec:
           command:
             - /ingestor
           args:
-            - "--kustainer"
             - "--storage-dir=/mnt/data"
             - "--max-segment-age=5s"
             - "--max-disk-usage=21474836480"

pkg/testutils/integration_test.go

@@ -0,0 +1,156 @@
+package testutils_test
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/Azure/adx-mon/pkg/testutils"
+	"github.com/Azure/adx-mon/pkg/testutils/collector"
+	"github.com/Azure/adx-mon/pkg/testutils/ingestor"
+	"github.com/Azure/adx-mon/pkg/testutils/kustainer"
+	"github.com/stretchr/testify/require"
+	"github.com/testcontainers/testcontainers-go"
+	"github.com/testcontainers/testcontainers-go/modules/k3s"
+)
+
+func TestIntegration(t *testing.T) {
+	// An extra generous timeout for the test. The test should run in
+	// about 5 minutes, but when running with the race detector, it
+	// can take longer.
+	wg := sync.WaitGroup{}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Hour)
+	t.Cleanup(cancel)
+
+	kustainerUrl := StartCluster(ctx, t)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		VerifyLogs(ctx, t, kustainerUrl)
+	}()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		VerifyMetrics(ctx, t, kustainerUrl)
+	}()
+
+	wg.Wait()
+}
+
+func StartCluster(ctx context.Context, t *testing.T) (kustoUrl string) {
+	t.Helper()
+	wg := sync.WaitGroup{}
+
+	k3sContainer, err := k3s.Run(ctx, "rancher/k3s:v1.31.2-k3s1")
+	testcontainers.CleanupContainer(t, k3sContainer)
+	require.NoError(t, err)
+
+	kustoContainer, err := kustainer.Run(ctx, "mcr.microsoft.com/azuredataexplorer/kustainer-linux:latest", kustainer.WithCluster(ctx, k3sContainer))
+	testcontainers.CleanupContainer(t, kustoContainer)
+	require.NoError(t, err)
+
+	restConfig, _, err := testutils.GetKubeConfig(ctx, k3sContainer)
+	require.NoError(t, err)
+	require.NoError(t, kustoContainer.PortForward(ctx, restConfig))
+
+	kubeconfig, err := testutils.WriteKubeConfig(ctx, k3sContainer, t.TempDir())
+	require.NoError(t, err)
+	t.Logf("Kubeconfig: %s", kubeconfig)
+	t.Logf("Kustainer: %s", kustoContainer.ConnectionUrl())
+
+	wg.Add(1)
+	go t.Run("Configure Kusto", func(t *testing.T) {
+		defer wg.Done()
+
+		opts := kustainer.IngestionBatchingPolicy{
+			MaximumBatchingTimeSpan: 30 * time.Second,
+		}
+		for _, dbName := range []string{"Metrics", "Logs"} {
+			require.NoError(t, kustoContainer.CreateDatabase(ctx, dbName))
+			require.NoError(t, kustoContainer.SetIngestionBatchingPolicy(ctx, dbName, opts))
+		}
+	})
+
+	wg.Add(1)
+	go t.Run("Build and install Ingestor", func(tt *testing.T) {
+		defer wg.Done()
+
+		ingestorContainer, err := ingestor.Run(ctx, ingestor.WithCluster(ctx, k3sContainer))
+		testcontainers.CleanupContainer(t, ingestorContainer)
+		require.NoError(tt, err)
+	})
+
+	wg.Add(1)
+	go t.Run("Build and install Collector", func(tt *testing.T) {
+		defer wg.Done()
+
+		collectorContainer, err := collector.Run(ctx, collector.WithCluster(ctx, k3sContainer))
+		testcontainers.CleanupContainer(t, collectorContainer)
+		require.NoError(tt, err)
+	})
+
+	kustoUrl = kustoContainer.ConnectionUrl()
+	wg.Wait()
+	return
+}
+
+func VerifyLogs(ctx context.Context, t *testing.T, kustainerUrl string) {
+	t.Helper()
+	var (
+		pollInterval = time.Second
+		timeout      = 5 * time.Minute
+		database     = "Logs"
+		table        = "Collector"
+	)
+
+	t.Run("Verify Logs", func(t *testing.T) {
+		t.Run("Table exists in Kusto", func(t *testing.T) {
+			require.Eventually(t, func() bool {
+				return testutils.TableExists(ctx, t, database, table, kustainerUrl)
+			}, timeout, pollInterval)
+		})
+
+		t.Run("Table has rows", func(t *testing.T) {
+			require.Eventually(t, func() bool {
+				return testutils.TableHasRows(ctx, t, database, table, kustainerUrl)
+			}, timeout, pollInterval)
+		})
+
+		t.Run("View exists in Kusto", func(t *testing.T) {
+			require.Eventually(t, func() bool {
+				return testutils.FunctionExists(ctx, t, database, table, kustainerUrl)
+			}, timeout, pollInterval)
+		})
+
+		t.Run("Verify view schema", func(t *testing.T) {
+			testutils.VerifyTableSchema(ctx, t, database, table, kustainerUrl, &collector.KustoTableSchema{})
+		})
+	})
+}
+
+func VerifyMetrics(ctx context.Context, t *testing.T, kustainerUrl string) {
+	t.Helper()
+	var (
+		pollInterval = time.Second
+		timeout      = 5 * time.Minute
+		database     = "Metrics"
+		table        = "AdxmonCollectorHealthCheck"
+	)
+
+	t.Run("Verify Metrics", func(t *testing.T) {
+		t.Run("Table exists in Kusto", func(t *testing.T) {
+			require.Eventually(t, func() bool {
+				return testutils.TableExists(ctx, t, database, table, kustainerUrl)
+			}, timeout, pollInterval)
+		})
+
+		t.Run("Table has rows", func(t *testing.T) {
+			require.Eventually(t, func() bool {
+				return testutils.TableHasRows(ctx, t, database, table, kustainerUrl)
+			}, timeout, pollInterval)
+		})
+	})
+}