Kafka3.7.0 安装部署
1.安装java环境
- 查看当前环境是否自带jdk，需要卸载旧版本
[root@tike ~]# rpm -qa | grep jkd //查看 [root@tike ~]rpm -e | grep java //删除 # 卸载 -e --nodeps 强制删除 [root@tike ~]# rpm -e --nodeps jdkxxxxxxxxxxxxxxxx
- 安装配置java-1.8-openjdk环境
[root@tike ~]# yum -y install java-1.8.0-openjdk [root@tike ~]# yum install -y java-1.8.0-openjdk-devel [root@tike ~]# java -version openjdk version "1.8.0_302" OpenJDK Runtime Environment (build 1.8.0_302-b08) OpenJDK 64-Bit Server VM (build 25.302-b08, mixed mode) // 不需要手工加 [root@tike ~]# vi /etc/profile # java environment JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.292.b10-1.el7_9.x86_64/jre PATH=$PATH:$JAVA_HOME/bin CLASSPATH=.:$JAVA_HOME/lib export JAVA_HOME CLASSPATH PATH [root@tike ~]# source /etc/profile
2.安装zookeeper集群
修改zoo.cfg文件，数据存放目录
[root@tike ~]# cd /tmp [root@tike ~]# wget https://mirrors.cnnic.cn/apache/zookeeper/zookeeper-3.7.0/apache-zookeeper-3.7.0-bin.tar.gz [root@tike ~]# tar -zxf apache-zookeeper-3.7.0-bin.tar.gz [root@tike ~]# mv apache-zookeeper-3.7.0-bin /usr/local/zookeeper [root@tike ~]# mkdir -p /data/zookeeper/data [root@tike ~]#cat>/usr/local/zookeeper/conf/zoo.cfg<<EOF tickTime=2000 initLimit=10 syncLimit=5 dataDir=/data/zookeeper/data server.1=10.5.2.143:2888:3888 server.2=10.5.2.141:2888:3888 server.3=10.5.2.142:2888:3888 clientPort=2181 EOF [root@tike ~]# echo 1 > /data/zookeeper/data/myid # 集群模式下需要新增一个名叫myid的文件，这个文件放在上述dataDir指定的目录下，这个文件里面就只有一个数据，就是配置文件中server.x的这个x（1,2,3）值，zookeeper启动时会读取这个文件，拿到里面的数据与 zoo.cfg 里面的配置信息比较从而判断到底是那个server（节点）。加入systemd管理
[root@tike ~]# cat>/etc/systemd/system/zookeeper.service<<EOF [Unit] Description=zookeeper.service After=network.target ConditionPathExists=/usr/local/zookeeper/conf/zoo.cfg [Service] Type=forking Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/jvm/jre-1.8.0-openjdk/bin" User=root Group=root ExecStart=/usr/local/zookeeper/bin/zkServer.sh start ExecStop=/usr/local/zookeeper/bin/zkServer.sh stop [Install] WantedBy=multi-user.target EOF启动
[root@tike ~]# systemctl daemon-reload [root@tike ~]# systemctl enable zookeeper [root@tike ~]# systemctl start zookeeper验证
[root@tike ~]# /usr/local/zookeeper/bin/zkCli.sh [root@tike ~]# /usr/local/zookeeper/bin/zkCli.sh -server 127.0.0.1:2181
3.zk脚本
zk-install.sh
#!/bin/bash if [ ! -n $1 ]; then echo "主机编号为必须参数" exit 1 fi cd /tmp rm -f apache-zookeeper-3.7.0.tar.gz rm -rf apache-zookeeper-3.7.0 rm -rf /usr/local/zookeeper wget https://mirrors.cnnic.cn/apache/zookeeper/zookeeper-3.7.0/apache-zookeeper-3.7.0-bin.tar.gz tar -zxf apache-zookeeper-3.7.0-bin.tar.gz mv apache-zookeeper-3.7.0-bin /usr/local/zookeeper mkdir -p /data/zookeeper/data cat>/usr/local/zookeeper/conf/zoo.cfg<<EOF tickTime=2000 initLimit=10 syncLimit=5 dataDir=/data/zookeeper/data server.1=172.30.26.21:2888:3888 server.2=172.30.26.22:2888:3888 server.3=172.30.26.23:2888:3888 clientPort=2181 EOF echo $1 > /data/zookeeper/data/myid cat /usr/local/zookeeper/conf/zoo.cfg cat /data/zookeeper/data/myid cd /usr/local/zookeeper cat>/etc/systemd/system/zookeeper.service<<EOF [Unit] Description=zookeeper.service After=network.target ConditionPathExists=/usr/local/zookeeper/conf/zoo.cfg [Service] Type=forking Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/jvm/jre-1.8.0-openjdk/bin" User=root Group=root ExecStart=/usr/local/zookeeper/bin/zkServer.sh start ExecStop=/usr/local/zookeeper/bin/zkServer.sh stop [Install] WantedBy=multi-user.target EOF systemctl daemon-reload systemctl enable zookeeper systemctl start zookeeper /usr/local/zookeeper/bin/zkServer.sh status
4.安装kafka
[root@tike ~]# cd /tmp [root@tike ~]# wget https://wu-devops.oss-cn-huhehaote.aliyuncs.com/kafka_2.13-2.7.0.tgz [root@tike ~]# tar -zxf kafka_2.13-2.7.0.tgz [root@tike ~]# mv kafka_2.13-2.7.0 /usr/local/kafka [root@tike ~]# mkdir -p /data/kafka/data写入配置文件
#!/bin/bash ip=$(ifconfig eth0 | grep "inet 1" | awk '{print $2}') id=$(echo $ip | awk -F "." '{print $4}') cat>/usr/local/kafka/config/cluster.properties<<EOF broker.id=$id port=9092 host.name=$ip delete.topic.enable=false log.dirs=/data/kafka/data zookeeper.connect=10.5.2.141:2181,10.5.2.142:2181,10.5.2.143:2181 zookeeper.connection.timeout.ms=6000 log.retention.hours=720 EOF验证
[root@tike ~]# cat /usr/local/kafka/config/cluster.properties加入systemd管理
[root@tike ~]# cat>/etc/systemd/system/kafka.service<[Unit]
Description=kafka.service
After=zookeeper.target
ConditionPathExists=/usr/local/kafka/config/cluster.properties
[Service]
Type=forking
Environment="KAFKA_HEAP_OPTS=-Xmx4G -Xms4G -Xmn2G"
Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/jvm/jre-1.8.0-openjdk/bin"
User=root
Group=root
ExecStart=/usr/local/kafka/bin/kafka-server-start.sh -daemon /usr/local/kafka/config/cluster.properties
ExecStop=/usr/local/kafka/bin/kafka-server-stop.sh
[Install]
WantedBy=multi-user.target
EOF由于担心日志打满当前目录，将服务日志输出到指定目录并定义一个变量，指定的值为系统日志输出路径，重启broker即可生效。
打开$KAFKA_HOME/bin/kafka-run-class.sh，
base_dir=$(dirname $0)/.. 下面
LOG_DIR=/data/kafka/serverlogs
启动Kafka
[root@tike ~]# systemctl daemon-reload
[root@tike ~]# systemctl enable kafka
[root@tike ~]# systemctl stop kafka
[root@tike ~]# systemctl start kafka
[root@tike ~]# systemctl status kafka
5.kafka脚本
#!/bin/bash
if [ ! -n $1 ]; then
echo "主机编号为必须参数"
exit 1
fi
cd /tmp
rm -f kafka_2.13-2.7.0.tgz
rm -rf kafka_2.13-2.7.0
rm -rf /usr/local/kafka
wget https://mirrors.cnnic.cn/apache/kafka/2.7.0/kafka_2.13-2.7.0.tgz
tar -zxf kafka_2.13-2.7.0.tgz
mv kafka_2.13-2.7.0 /usr/local/kafka
mkdir -p /data/kafka/data
ip=$(ifconfig eth0 | grep "inet 1" | awk '{print $2}')
id=$(echo $ip | awk -F "." '{print $4}')
cat>/usr/local/kafka/config/cluster.properties<broker.id=$id
port=9092
host.name=$ip
delete.topic.enable=false
log.dirs=/data/kafka/data
zookeeper.connect=172.30.26.21:2181,172.30.26.22:2181,172.30.26.23:2181
zookeeper.connection.timeout.ms=6000
log.retention.hours=720
EOF
cat>/etc/systemd/system/kafka.service<[Unit]
Description=kafka.service
After=zookeeper.target
ConditionPathExists=/usr/local/kafka/config/cluster.properties
[Service]
Type=forking
Environment="KAFKA_HEAP_OPTS=-Xmx4G -Xms4G -Xmn2G"
Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/jvm/jre-1.8.0-openjdk/bin"
User=root
Group=root
ExecStart=/usr/local/kafka/bin/kafka-server-start.sh -daemon /usr/local/kafka/config/cluster.properties
ExecStop=/usr/local/kafka/bin/kafka-server-stop.sh
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable kafka
systemctl stop kafka
systemctl start kafka
systemctl status kafka
6.kafka-manager
安装
[root@tike ~]# cd /tmp
[root@tike ~]# wget https://wu-devops.oss-cn-huhehaote.aliyuncs.com/kafka-manager-1.3.3.18.tar.gz
[root@tike ~]# mv kafka-manager-1.3.3.18 /usr/local/kafka-manager
[root@tike ~]# mkdir -p /data/kafka-manager/logs
[root@tike ~]# kafka-manager.zkhosts="10.5.2.141:2181,10.5.2.142:2181,10.5.2.143:2181"修改配置文件
play.crypto.secret="^play.crypto.secret=${?APPLICATION_SECRET}
# The application languages
# ~~~~~
play.i18n.langs=["en"]
play.http.requestHandler = "play.http.DefaultHttpRequestHandler"
play.http.context = "/"
play.application.loader=loader.KafkaManagerLoader
kafka-manager.zkhosts="10.5.2.141:2181,10.5.2.142:2181,10.5.2.143:2181"
#kafka-manager.zkhosts="kafka-manager-zookeeper:2181"
#kafka-manager.zkhosts=${?ZK_HOSTS}
pinned-dispatcher.type="PinnedDispatcher"
pinned-dispatcher.executor="thread-pool-executor"
application.features=["KMClusterManagerFeature","KMTopicManagerFeature","KMPreferredReplicaElectionFeature","KMReassignPartitionsFeature"]
akka {
loggers = ["akka.event.slf4j.Slf4jLogger"]
loglevel = "INFO"
}
akka.logger-startup-timeout = 60s
basicAuthentication.enabled=false
basicAuthentication.enabled=${?KAFKA_MANAGER_AUTH_ENABLED}
basicAuthentication.username="admin"
basicAuthentication.username=${?KAFKA_MANAGER_USERNAME}
basicAuthentication.password="password"
basicAuthentication.password=${?KAFKA_MANAGER_PASSWORD}
basicAuthentication.realm="Kafka-Manager"
basicAuthentication.excluded=["/api/health"] # ping the health of your instance without authentification
kafka-manager.consumer.properties.file=${?CONSUMER_PROPERTIES_FILE}加入systemctl管理
启动脚本
cd /usr/local/kafka-manager/bin/
vim start.sh
#!/bin/sh
nohup /usr/local/kafka-manager/bin/kafka-manager \
-Dconfig.file=/usr/local/kafka-manager/conf/application.conf \
>/data/kafka-manager/logs/kafka-manager.log 2>&1写入kafka-manager.service
[root@tike ~]# cat>/etc/systemd/system/kafka-manager.service<[Unit]
Description=kafka-manager.service
After=zookeeper.target
ConditionPathExists=/usr/local/kafka-manager/conf/application.conf
[Service]
Type=simple
Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/jvm/jre-1.8.0-openjdk/bin"
User=root
Group=root
ExecStart=/usr/local/kafka-manager/bin/start.sh
KillSignal=SIGTERM
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF