kafka批量修改Topic副本

Kafka 批量修改topic副本

列出副本为1的topic

topic.sh

topics=`/opt/server/kafka/bin/kafka-topics.sh --zookeeper 192.168.25.123:2181 -list`

for i in $topics;do
  replicsNum=`/opt/server/kafka/bin/kafka-topics.sh --zookeeper 192.168.25.123:2181 -describe --topic $i|grep ReplicationFactor|awk '{print$3}'|awk -F: '{print $2}'`
  if [ $replicsNum == 1 ];then
     echo  $i >> /opt/server/kafka/bin/topic.txt
  fi
done

leaders.txt

填写broker id

1
2
3

生成topic配置json

topic-reassignment.sh

#!/bin/bash
ZK_HOST="192.168.25.123:2181"

topics=`cat ./topic.txt`

IFS=$'\n'
echo '{"version":1,"partitions":[' > topic-reassignment.json
for i in $topics;do
  echo 'write file for '$i''
  leaders=`./kafka-topics.sh --zookeeper $ZK_HOST -describe --topic $i|grep Leader`
  for leader in $leaders;do
    partition=`echo $leader |awk '{print $4}'`
    leader=`echo $leader |awk '{print $6}'`
    # 如果需要保证 副分片 和 主分片 不在同一个 Broker 且随机分配在剩余的节点中，可以使用
    # follwer=`grep -vxF $leader ./leaders.txt | shuf -n1`
    follwer=`grep -vxF $leader ./leaders.txt`
    follwer1=`echo $follwer | awk '{print $1}'`
    follwer2=`echo $follwer | awk '{print $2}'`
    echo '{"topic":"'$i'","partition":'$partition',"replicas":['$leader','$follwer1','$follwer2']},' >> topic-reassignment.json
  done
done
echo ']}' >> topic-reassignment.json

扩展副本

./kafka-reassign-partitions.sh --zookeeper 192.168.25.123:2181 --reassignment-json-file topic-reassignment.json --execute