Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

tutorial: Deploying Kafka on Kubernetes using Strimzi and Pulumi #13818

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
507 changes: 507 additions & 0 deletions content/tutorials/kubernetes-strimzi-kafka/index.md

Large diffs are not rendered by default.

Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
188 changes: 188 additions & 0 deletions static/programs/strimzi-kafka-csharp/Program.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
using Pulumi;
using Pulumi.Kubernetes.Helm.V3;
using Pulumi.Kubernetes.ApiExtensions;
using Pulumi.Kubernetes.Types.Inputs.Helm.V3;
using Pulumi.Kubernetes.Types.Inputs.Meta.V1;
using System.Threading.Tasks;

class KafKaNodePoolArgs : CustomResourceArgs {
public KafKaNodePoolArgs() : base("kafka.strimzi.io/v1beta2", "KafkaNodePool") {}
[Input("spec")]
public Input<KafakNodePoolSpecArgs> Spec { get; set; }
}

class KafakNodePoolSpecArgs : ResourceArgs {
public Input<string> Replicas { get; set; }
public InputList<string> Roles { get; set; }
public InputMap<object> Storage { get; set; }
}

class KafkaClusterArgs : CustomResourceArgs {
public KafkaClusterArgs() : base("kafka.strimzi.io/v1beta2", "Kafka") {}
[Input("spec")]
public Input<KafakaClusterSpecArgs> Spec { get; set; }
}

class KafakaClusterSpecArgs : ResourceArgs {
public InputMap<object> Kafka { get; set; }
public InputMap<object> EntityOperator { get; set; }
}

class KafkaUserArgs : CustomResourceArgs {
public KafkaUserArgs() : base("kafka.strimzi.io/v1beta2", "KafkaUser") {}
[Input("spec")]
public Input<KafkaUserSpecArgs> Spec { get; set; }
}

class KafkaUserSpecArgs : ResourceArgs {
public InputMap<object> Authentication { get; set; }
public InputMap<object> Authorization { get; set; }
}

class KafkaTopicArgs : CustomResourceArgs {
public KafkaTopicArgs() : base("kafka.strimzi.io/v1beta2", "KafkaTopic") {}
[Input("spec")]
public Input<KafkaTopicSpecArgs> Spec { get; set; }
}

class KafkaTopicSpecArgs : ResourceArgs {
public Input<int> Partitions { get; set; }
public Input<int> Replicas { get; set; }
public InputMap<object> Config { get; set; }
}

class ListenerArgs : ResourceArgs {
public Input<string> Name { get; set; }
public Input<int> Port { get; set; }
public Input<string> Type { get; set; }
public Input<bool> Tls { get; set; }
}

class MyStack : Stack {
public MyStack() {
// Deploy Strimzi Kafka operator using Helm chart
var strimzi = new Release("strimzi-kafka-operator", new ReleaseArgs {
Chart = "oci://quay.io/strimzi-helm/strimzi-kafka-operator",
Namespace = "strimzi",
CreateNamespace = true,
Values =
new InputMap<object> {
{ "watchAnyNamespace", true },
},
});

// Deploy Kafka Node Pool
var kafkaNodePool = new Pulumi.Kubernetes.ApiExtensions.CustomResource(
"kafkaNodePool",
new KafKaNodePoolArgs {
Metadata = new ObjectMetaArgs { Name = "my-kafka-nodepool",
Labels = { { "strimzi.io/cluster",
"my-cluster" } } },
Spec =
new KafakNodePoolSpecArgs {
Replicas = "1",
Roles = new InputList<string> { "controller", "broker" },
Storage = new InputMap<object> { { "type", "ephemeral" } }
},
},
new CustomResourceOptions { DependsOn = { strimzi } });

// Deploy Kafka Cluster
var kafkaCluster = new Pulumi.Kubernetes.ApiExtensions.CustomResource(
"kafkaCluster",
new KafkaClusterArgs {
Metadata =
new ObjectMetaArgs { Name = "my-cluster",
Annotations = { { "strimzi.io/kraft", "enabled" },
{ "strimzi.io/node-pools",
"enabled" } } },
Spec =
new KafakaClusterSpecArgs {
Kafka =
new InputMap<object> {
{ "version", "3.8.0" },
{"replicas", 1 },
{ "storage",
new InputMap<object> { { "type", "ephemeral" } } },
{ "listeners",
new InputList<ListenerArgs> { new ListenerArgs() {
Name = "plain",
Port = 9092,
Type = "internal",
Tls = false,
},
new ListenerArgs() {
Name = "tls",
Port = 9093,
Type = "internal",
Tls = true,
} } },
{ "metadataVersion", "3.8-IV0" },
{ "config",
new InputMap<object> {
{ "offsets.topic.replication.factor", "1" },
{ "transaction.state.log.replication.factor", "1" },
{ "transaction.state.log.min.isr", "1" },
{ "default.replication.factor", "1" },
{ "min.insync.replicas", "1" },
} }
},
EntityOperator =
new InputMap<object> {
{ "topicOperator", new InputMap<object>() },
{ "userOperator", new InputMap<object>() }
}
},
},
new CustomResourceOptions { DependsOn = { strimzi } });

// Deploy Kafka Topic
var kafkaTopic = new Pulumi.Kubernetes.ApiExtensions.CustomResource(
"kafkaTopic",
new KafkaTopicArgs {
Metadata = new ObjectMetaArgs { Name = "my-topic",
Labels = { { "strimzi.io/cluster",
"my-cluster" } } },
Spec =
new KafkaTopicSpecArgs {
Partitions = 5, Replicas = 1,
Config =
new InputMap<object> { { "retention.ms", "7200000" },
{ "segment.bytes", "1073741824" } }
},
},
new CustomResourceOptions { DependsOn = { kafkaCluster } });

// Deploy Kafka User
var kafkaUser =
new Pulumi.Kubernetes.ApiExtensions.CustomResource(
"kafkaUser",
new KafkaUserArgs {
Metadata = new ObjectMetaArgs { Name = "my-kafka-user",
Labels = { { "strimzi.io/cluster",
"my-cluster" } } },
Spec =
new KafkaUserSpecArgs {
Authentication =
new InputMap<object> { { "type", "scram-sha-512" } },
Authorization =
new InputMap<object> {
{ "type", "simple" },
{ "acls", new[] { new InputMap<object> {
{ "resource",
new InputMap<object> { { "type", "topic" },
{ "name", "my-topic" },
{ "patternType",
"literal" } } },
{ "operation", "Read" },
{ "host", "*" }
} } }
}
},
},
new CustomResourceOptions { DependsOn = { kafkaCluster } });
}
}
class Program {
static Task<int> Main() => Deployment.RunAsync<MyStack>();
}
7 changes: 7 additions & 0 deletions static/programs/strimzi-kafka-csharp/Pulumi.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
name: strimzi-kafka-csharp
description: Example on how to deploy Strimzi Kafka
runtime: dotnet
config:
pulumi:tags:
value:
pulumi:template: kubernetes-csharp
14 changes: 14 additions & 0 deletions static/programs/strimzi-kafka-csharp/strimzi-kafka-csharp.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Pulumi" Version="3.*" />
<PackageReference Include="Pulumi.Kubernetes" Version="4.*" />
</ItemGroup>

</Project>
7 changes: 7 additions & 0 deletions static/programs/strimzi-kafka-go/Pulumi.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
name: strimzi-kafka-go
description: Example on how to deploy Strimzi Kafka
runtime: go
config:
pulumi:tags:
value:
pulumi:template: kubernetes-go
93 changes: 93 additions & 0 deletions static/programs/strimzi-kafka-go/go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
module strimzi-kafka-go

go 1.21

toolchain go1.23.4

require (
github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.12.0
github.com/pulumi/pulumi/sdk/v3 v3.117.0
)

require (
dario.cat/mergo v1.0.0 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/ProtonMail/go-crypto v1.0.0 // indirect
github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect
github.com/agext/levenshtein v1.2.3 // indirect
github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
github.com/atotto/clipboard v0.1.4 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/blang/semver v3.5.1+incompatible // indirect
github.com/charmbracelet/bubbles v0.16.1 // indirect
github.com/charmbracelet/bubbletea v0.24.2 // indirect
github.com/charmbracelet/lipgloss v0.7.1 // indirect
github.com/cheggaaa/pb v1.0.29 // indirect
github.com/cloudflare/circl v1.3.7 // indirect
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/djherbis/times v1.5.0 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.5.0 // indirect
github.com/go-git/go-git/v5 v5.12.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/glog v1.2.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/hcl/v2 v2.17.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-localereader v0.0.1 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/mitchellh/go-ps v1.0.0 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
github.com/muesli/cancelreader v0.2.2 // indirect
github.com/muesli/reflow v0.3.0 // indirect
github.com/muesli/termenv v0.15.2 // indirect
github.com/opentracing/basictracer-go v1.1.0 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pgavlin/fx v0.1.6 // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pkg/term v1.1.0 // indirect
github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 // indirect
github.com/pulumi/esc v0.6.2 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/rogpeppe/go-internal v1.11.0 // indirect
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 // indirect
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
github.com/skeema/knownhosts v1.2.2 // indirect
github.com/spf13/cobra v1.8.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/texttheater/golang-levenshtein v1.0.1 // indirect
github.com/tweekmonster/luser v0.0.0-20161003172636-3fa38070dbd7 // indirect
github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/zclconf/go-cty v1.13.2 // indirect
go.uber.org/atomic v1.9.0 // indirect
golang.org/x/crypto v0.23.0 // indirect
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
golang.org/x/mod v0.14.0 // indirect
golang.org/x/net v0.25.0 // indirect
golang.org/x/sync v0.6.0 // indirect
golang.org/x/sys v0.20.0 // indirect
golang.org/x/term v0.20.0 // indirect
golang.org/x/text v0.15.0 // indirect
golang.org/x/tools v0.17.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 // indirect
google.golang.org/grpc v1.63.2 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/frand v1.4.2 // indirect
)
Loading
Loading