Skip to content

Instantly share code, notes, and snippets.

@Nurlan199206
Last active January 21, 2024 06:44
Show Gist options
  • Save Nurlan199206/c7438626b5f122ae8343fe9ec528c517 to your computer and use it in GitHub Desktop.
Save Nurlan199206/c7438626b5f122ae8343fe9ec528c517 to your computer and use it in GitHub Desktop.
KAFKA & ZOOKEEPER cluster
1. 10.204.11.181
2. 10.204.11.182
3. 10.204.11.183
wget https://downloads.apache.org/kafka/3.6.1/kafka_2.13-3.6.1.tgz
wget https://dlcdn.apache.org/zookeeper/zookeeper-3.9.1/apache-zookeeper-3.9.1-bin.tar.gz
tar xvzf kafka_2.13-3.6.1.tgz
tar xvzf apache-zookeeper-3.9.1-bin.tar.gz
mv kafka_2.13-3.6.1.tgz /opt/kafka
mv apache-zookeeper-3.9.1-bin.tar.gz /opt/zookeeper
====================================================web-ui=================================================
apt install docker.io
docker run -d -it -p 8085:8080 -e DYNAMIC_CONFIG_ENABLED=true provectuslabs/kafka-ui:master
==============================================KAFKA SYSTEMD================================================
[Unit]
Requires=kafka.service
After=kafka.service
[Service]
Type=simple
User=root
ExecStart=/bin/sh -c '/opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties'
ExecStop=/opt/kafka/bin/kafka-server-stop.sh
Restart=on-abnormal
[Install]
WantedBy=multi-user.target
===============================================ZOOKEEPER SYSTEMD===========================================
[Unit]
Description=Zookeeper Daemon
Documentation=http://zookeeper.apache.org
Requires=network.target
After=network.target
[Service]
Type=forking
WorkingDirectory=/opt/zookeeper
User=root
Group=root
ExecStart=/opt/zookeeper/bin/zkServer.sh start /opt/zookeeper/conf/zoo.cfg
ExecStop=/opt/zookeeper/bin/zkServer.sh stop /opt/zookeeper/conf/zoo.cfg
ExecReload=/opt/zookeeper/bin/zkServer.sh restart /opt/zookeeper/conf/zoo.cfg
TimeoutSec=30
Restart=on-failure
[Install]
WantedBy=default.target
=============================================srv-01 /opt/kafka/config/server.properties==========================================
broker.id=1
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
# The maximum size of a request that the socket server will accept (protection against OOM)
#socket.request.max.bytes=104857600
log.dirs=/tmp/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
listeners=PLAINTEXT://:9092
advertised.host.name=10.204.11.181
advertised.port=9092
advertised.listeners=PLAINTEXT://10.204.11.181:9092
zookeeper.connect=10.204.11.181:2181,10.204.11.182:2181,10.204.11.183:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0
=============================================srv-02 /opt/kafka/config/server.properties==========================================
broker.id=2
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
# The maximum size of a request that the socket server will accept (protection against OOM)
socket.request.max.bytes=104857600
log.dirs=/tmp/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=30000
listeners=PLAINTEXT://:9092
advertised.host.name=10.204.11.182
advertised.port=9092
advertised.listeners=PLAINTEXT://10.204.11.182:9092
zookeeper.connect=10.204.11.181:2181,10.204.11.182:2181,10.204.11.183:2181
zookeeper.connection.timeout.ms=18000
=============================================srv-03 /opt/kafka/config/server.properties==========================================
broker.id=3
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
# The maximum size of a request that the socket server will accept (protection against OOM)
socket.request.max.bytes=104857600
log.dirs=/tmp/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
advertised.host.name=10.204.11.183
listeners=PLAINTEXT://:9092
advertised.listeners=PLAINTEXT://10.204.11.183:9092
zookeeper.connect=10.204.11.181:2181,10.204.11.182:2181,10.204.11.183:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0
======================================srv-01 /opt/zookeeper/conf/zoo.cfg==========================================================
tickTime=2000
dataDir=/var/lib/zookeeper
clientPort=2181
initLimit=5
syncLimit=2
server.1=10.204.11.181:2888:3888
server.2=10.204.11.182:2888:3888
server.3=10.204.11.183:2888:3888
======================================srv-02 /opt/zookeeper/conf/zoo.cfg==========================================================
tickTime=2000
dataDir=/var/lib/zookeeper
clientPort=2181
initLimit=5
syncLimit=2
server.1=10.204.11.181:2888:3888
server.2=10.204.11.182:2888:3888
server.3=10.204.11.183:2888:3888
======================================srv-03 /opt/zookeeper/conf/zoo.cfg==========================================================
tickTime=2000
dataDir=/var/lib/zookeeper
clientPort=2181
initLimit=5
syncLimit=2
server.1=10.204.11.181:2888:3888
server.2=10.204.11.182:2888:3888
server.3=10.204.11.183:2888:3888
==========================================================zookeeper node id=============================================================
mkdir -p /var/lib/zookeeper/
touch /var/lib/zookeeper/myid
echo "1" > /var/lib/zookeeper/myid
echo "2" > /var/lib/zookeeper/myid
echo "3" > /var/lib/zookeeper/myid
========================================================================================================================================
systemctl start zookeeper
systemctl start kafka
systemctl status kafka
systemctl status zookeeper
systemctl enable kafka
systemctl enable zookeeper
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment