Deploy a Kafka 3 nodes Cluster in few minutes on OVHcloud multi-regions.
In this example, we’ll deploy the Kafka in cluster Mode (KRaft - Apache Kafka Without ZooKeeper).
Here we’ll look at how to deploy 4 services with a single YAML file:
- 3 controllers and brokers
- 1 kafka-ui (web interface)
services:
- id: kafka1
name: kafka-1
dockerConfiguration:
image: apache/kafka
imageVersion: "latest"
hostname: "kafka-1"
countMin: 1
countMax: 1
cpuLimit: 512 # MHz
memoryLimitMiB: 512
ports:
- listeningPort: 9092
- listeningPort: 9093
environmentVariables:
- key: KAFKA_NODE_ID
value: "1"
- key: KAFKA_PROCESS_ROLES
value: "controller,broker"
- key: KAFKA_CONTROLLER_QUORUM_VOTERS
value: "1@127.0.0.1:9093,2@%KAFKA2_ADD%,3@%KAFKA3_ADD%"
- key: KAFKA_KRAFT_CLUSTER_ID
value: "H0DqvlGnSvOx47E_4f9w1w"
- key: KAFKA_LISTENERS
value: "CONTROLLER://0.0.0.0:9093,CONNECTIONS_FROM_HOST://0.0.0.0:9092"
- key: KAFKA_ADVERTISED_LISTENERS
value: "CONNECTIONS_FROM_HOST://127.0.0.1:19092"
- key: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
value: "CONTROLLER:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT"
- key: KAFKA_CONTROLLER_LISTENER_NAMES
value: "CONTROLLER"
- key: KAFKA_INTER_BROKER_LISTENER_NAME
value: "CONNECTIONS_FROM_HOST"
- key: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR
value: "3"
- key: KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR
value: "3"
- key: KAFKA_TRANSACTION_STATE_LOG_MIN_ISR
value: "2"
links:
- toServiceId: kafka2
toServicePort: 9093
localExposedPort: 29093
variableHost: KAFKA2_HOST
variablePort: KAFKA2_PORT
variableAddress: KAFKA2_ADD
- toServiceId: kafka2
toServicePort: 9092
localExposedPort: 29092
variableHost: KAFKA2_HOST2
variablePort: KAFKA2_PORT2
variableAddress: KAFKA2_ADD2
- toServiceId: kafka3
toServicePort: 9093
localExposedPort: 39093
variableHost: KAFKA3_HOST
variablePort: KAFKA3_PORT
variableAddress: KAFKA3_ADD
- toServiceId: kafka3
toServicePort: 9092
localExposedPort: 39092
variableHost: KAFKA3_HOST2
variablePort: KAFKA3_PORT2
variableAddress: KAFKA3_ADD2
- id: kafka2
name: kafka-2
dockerConfiguration:
image: apache/kafka
imageVersion: "latest"
hostname: "kafka-2"
countMin: 1
countMax: 1
cpuLimit: 512 # MHz
memoryLimitMiB: 512
ports:
- listeningPort: 9092
- listeningPort: 9093
environmentVariables:
- key: KAFKA_NODE_ID
value: "2"
- key: KAFKA_PROCESS_ROLES
value: "controller,broker"
- key: KAFKA_CONTROLLER_QUORUM_VOTERS
value: "1@%KAFKA1_ADD%,2@127.0.0.1:9093,3@%KAFKA3_ADD%"
- key: KAFKA_KRAFT_CLUSTER_ID
value: "H0DqvlGnSvOx47E_4f9w1w"
- key: KAFKA_LISTENERS
value: "CONTROLLER://0.0.0.0:9093,CONNECTIONS_FROM_HOST://0.0.0.0:9092"
- key: KAFKA_ADVERTISED_LISTENERS
value: "CONNECTIONS_FROM_HOST://127.0.0.1:29092"
- key: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
value: "PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT"
- key: KAFKA_CONTROLLER_LISTENER_NAMES
value: "CONTROLLER"
- key: KAFKA_INTER_BROKER_LISTENER_NAME
value: "CONNECTIONS_FROM_HOST"
- key: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR
value: "3"
- key: KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR
value: "3"
- key: KAFKA_TRANSACTION_STATE_LOG_MIN_ISR
value: "2"
links:
- toServiceId: kafka1
toServicePort: 9093
localExposedPort: 19093
variableHost: KAFKA1_HOST
variablePort: KAFKA1_PORT
variableAddress: KAFKA1_ADD
- toServiceId: kafka1
toServicePort: 9092
localExposedPort: 19092
variableHost: KAFKA1_HOST2
variablePort: KAFKA1_PORT2
variableAddress: KAFKA1_ADD2
- toServiceId: kafka3
toServicePort: 9093
localExposedPort: 39093
variableHost: KAFKA3_HOST
variablePort: KAFKA3_PORT
variableAddress: KAFKA3_ADD
- toServiceId: kafka3
toServicePort: 9092
localExposedPort: 39092
variableHost: KAFKA3_HOST2
variablePort: KAFKA3_PORT2
variableAddress: KAFKA3_ADD2
- id: kafka3
name: kafka-3
dockerConfiguration:
image: apache/kafka
imageVersion: "latest"
hostname: "kafka-3"
countMin: 1
countMax: 1
cpuLimit: 512 # MHz
memoryLimitMiB: 512
ports:
- listeningPort: 9092
- listeningPort: 9093
environmentVariables:
- key: KAFKA_NODE_ID
value: "3"
- key: KAFKA_PROCESS_ROLES
value: "controller,broker"
- key: KAFKA_CONTROLLER_QUORUM_VOTERS
value: "1@%KAFKA1_ADD%,2@%KAFKA2_ADD%,3@127.0.0.1:9093"
- key: KAFKA_KRAFT_CLUSTER_ID
value: "H0DqvlGnSvOx47E_4f9w1w"
- key: KAFKA_LISTENERS
value: "CONTROLLER://0.0.0.0:9093,CONNECTIONS_FROM_HOST://0.0.0.0:9092"
- key: KAFKA_ADVERTISED_LISTENERS
value: "CONNECTIONS_FROM_HOST://127.0.0.1:39092"
- key: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
value: "PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT"
- key: KAFKA_CONTROLLER_LISTENER_NAMES
value: "CONTROLLER"
- key: KAFKA_INTER_BROKER_LISTENER_NAME
value: "CONNECTIONS_FROM_HOST"
- key: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR
value: "3"
- key: KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR
value: "3"
- key: KAFKA_TRANSACTION_STATE_LOG_MIN_ISR
value: "2"
links:
- toServiceId: kafka1
toServicePort: 9093
localExposedPort: 19093
variableHost: KAFKA1_HOST
variablePort: KAFKA1_PORT
variableAddress: KAFKA1_ADD
- toServiceId: kafka1
toServicePort: 9092
localExposedPort: 19092
variableHost: KAFKA1_HOST2
variablePort: KAFKA1_PORT2
variableAddress: KAFKA1_ADD2
- toServiceId: kafka2
toServicePort: 9093
localExposedPort: 29093
variableHost: KAFKA2_HOST
variablePort: KAFKA2_PORT
variableAddress: KAFKA2_ADD
- toServiceId: kafka2
toServicePort: 9092
localExposedPort: 29092
variableHost: KAFKA2_HOST2
variablePort: KAFKA2_PORT2
variableAddress: KAFKA2_ADD2
- id: kafkaui
name: kafka-ui
dockerConfiguration:
image: provectuslabs/kafka-ui
imageVersion: "latest"
countMin: 1
countMax: 1
cpuLimit: 512 # MHz
memoryLimitMiB: 512
ports:
- listeningPort: 8080
loadBalancerRules:
- publicPort: 443
environmentVariables:
- key: DYNAMIC_CONFIG_ENABLED
value: "true"
links:
- toServiceId: kafka1
toServicePort: 9092
localExposedPort: 19092
variableHost: KAFKA1_HOST
variablePort: KAFKA1_PORT
variableAddress: KAFKA1_ADD
- toServiceId: kafka2
toServicePort: 9092
localExposedPort: 29092
variableHost: KAFKA2_HOST
variablePort: KAFKA2_PORT
variableAddress: KAFKA2_ADD
- toServiceId: kafka3
toServicePort: 9092
localExposedPort: 39092
variableHost: KAFKA3_HOST
variablePort: KAFKA3_PORT
variableAddress: KAFKA3_ADD
