{"id":406,"date":"2021-05-12T14:39:42","date_gmt":"2021-05-12T06:39:42","guid":{"rendered":"https:\/\/22pig.com\/?p=406"},"modified":"2021-05-12T14:39:42","modified_gmt":"2021-05-12T06:39:42","slug":"kafka-%e5%ae%89%e8%a3%85%e5%85%a5%e9%97%a8","status":"publish","type":"post","link":"http:\/\/22pig.com\/?p=406","title":{"rendered":"kafka \u5b89\u88c5\u5165\u95e8"},"content":{"rendered":"

kafka\u6b63\u5e38\u8fd0\u884c\uff0c\u5fc5\u987b\u914d\u7f6ezookeeper\uff0c\u5426\u5219\u65e0\u8bba\u662fkafka\u96c6\u7fa4\u8fd8\u662f\u5ba2\u6237\u7aef\u7684\u751f\u5b58\u8005\u548c\u6d88\u8d39\u8005\u90fd\u65e0\u6cd5\u6b63\u5e38\u7684\u5de5\u4f5c\u7684\uff1b\u6240\u4ee5\u9700\u8981\u914d\u7f6e\u542f\u52a8zookeeper\u670d\u52a1\u3002<\/p>\n

\uff081\uff09zookeeper\u9700\u8981java\u73af\u5883<\/p>\n

\n
<\/div>\n<\/div>\n

# yum -y install java-1.8.0<\/code><\/p>\n

\u89e3\u538b\u5230\u6307\u5b9a\u76ee\u5f55<\/p>\n

tar -<\/span>xzf kafka_2.13-2.8.0.<\/span>tgz -<\/span>C \/opt\/<\/span><\/pre>\n
\uff082\uff09\u8fd9\u91cckafka\u4e0b\u8f7d\u5305\u5df2\u7ecf\u5305\u62eczookeeper\u670d\u52a1\uff0c\u6240\u4ee5\u53ea\u9700\u4fee\u6539\u914d\u7f6e\u6587\u4ef6\uff0c\u542f\u52a8\u5373\u53ef\u3002<\/p>\n
\u5982\u679c\u9700\u8981\u4e0b\u8f7d\u6307\u5b9azookeeper\u7248\u672c\uff1b\u53ef\u4ee5\u5355\u72ec\u53bbzookeeper\u5b98\u7f51http:\/\/mirrors.shu.edu.cn\/apache\/zookeeper\/<\/a>\u4e0b\u8f7d\u6307\u5b9a\u7248\u672c\u3002<\/p>\n
\n
<\/div>\n<\/div>\n
[root@~]<\/code># cd \/data\/kafka_2.13-2.8.0\/<\/code><\/div>\n
[root@ kafka_2.11-2.1.0]<\/code># grep \"^[^#]\" config\/zookeeper.properties<\/code><\/div>\n
dataDir=<\/code>\/tmp\/zookeeper<\/code>\u00a0\u00a0\u00a0#\u6570\u636e\u5b58\u50a8\u76ee\u5f55<\/code><\/div>\n
clientPort=2181\u00a0\u00a0\u00a0<\/code>#zookeeper\u7aef\u53e3<\/code><\/div>\n
maxClientCnxns=0<\/code><\/div>\n
\u6ce8\uff1a\u53ef\u81ea\u884c\u6dfb\u52a0\u4fee\u6539zookeeper\u914d\u7f6e<\/p>\n
3.3\u00a0\u914d\u7f6ekafka<\/strong><\/h3>\n
\uff081\uff09\u4fee\u6539\u914d\u7f6e\u6587\u4ef6<\/p>\n
\n
\n
\n
[root@ kafka_2.13-2.8.0]<\/code># grep \"^[^#]\" config\/server.properties<\/code><\/div>\n
broker.<\/code>id<\/code>=0<\/code><\/div>\n
listeners=PLAINTEXT:<\/code>\/\/localhost<\/code>:9092<\/code><\/div>\n
num.network.threads=3<\/code><\/div>\n
num.io.threads=8<\/code><\/div>\n
socket.send.buffer.bytes=102400<\/code><\/div>\n
socket.receive.buffer.bytes=102400<\/code><\/div>\n
socket.request.max.bytes=104857600<\/code><\/div>\n
log.<\/code>dirs<\/code>=<\/code>\/tmp\/kafka-logs<\/code><\/div>\n
num.partitions=1<\/code><\/div>\n
num.recovery.threads.per.data.<\/code>dir<\/code>=1<\/code><\/div>\n
offsets.topic.replication.factor=1<\/code><\/div>\n
transaction.state.log.replication.factor=1<\/code><\/div>\n
transaction.state.log.min.isr=1<\/code><\/div>\n
log.retention.hours=168<\/code><\/div>\n
log.segment.bytes=1073741824<\/code><\/div>\n
log.retention.check.interval.ms=300000<\/code><\/div>\n
zookeeper.connect=localhost:2181<\/code><\/div>\n
zookeeper.connection.timeout.ms=6000<\/code><\/div>\n
group.initial.rebalance.delay.ms=0<\/code><\/div>\n<\/div>\n<\/div>\n<\/div>\n
\u6ce8\uff1a\u53ef\u6839\u636e\u81ea\u5df1\u9700\u6c42\u4fee\u6539\u914d\u7f6e\u6587\u4ef6<\/p>\n
    \n
  • \u00a0broker.id\uff1a\u552f\u4e00\u6807\u8bc6ID<\/li>\n
  • \u00a0listeners=PLAINTEXT:\/\/localhost:9092\uff1akafka\u670d\u52a1\u76d1\u542c\u5730\u5740\u548c\u7aef\u53e3<\/li>\n
  • \u00a0log.dirs\uff1a\u65e5\u5fd7\u5b58\u50a8\u76ee\u5f55<\/li>\n
  • \u00a0zookeeper.connect\uff1a\u6307\u5b9azookeeper\u670d\u52a1<\/li>\n<\/ul>\n
    \u2014\u2014\u2014\u2014\u2014\u2014\u2014\u2014\u2014\u2014\u2014\u2014\u2014\u2014\u2014\u2014\u2014\u2014\u2014\u2014\u2014\u2014\u2014\u2014<\/p>\n
    2\u3001\u542f\u52a8zk\uff08cd \/data\/kafka_2.13-2.8.0\/bin\uff09<\/p>\n
    .\/zookeeper-server-start.sh -daemon \/data\/kafka_2.13-2.8.0\/config\/zookeeper.properties<\/p>\n
    3\u3001\u4fee\u6539Kafka\u914d\u7f6e\uff0c\u5e76\u542f\u52a8<\/p>\n
    .\/kafka-server-start.sh -daemon \/data\/kafka_2.13-2.8.0\/config\/server.properties<\/p>\n
    4\u3001\u521b\u5efatopic<\/p>\n
    bin\/kafka-topics.sh –create –zookeeper localhost:2181 –replication-factor 1 –partitions 1 –topic test<\/p>\n
    5\u3001\u4f7f\u7528\u751f\u4ea7\u8005\u53d1\u9001\u6d88\u606f\uff0c\u6bcf\u884c\u662f\u4e00\u6761\u72ec\u7acb\u7684\u6d88\u606f<\/p>\n
    and\u00a0 go ….<\/p>\n
     <\/p>\n
    # \u521b\u5efatopic
    \nkafka-topics.sh –zookeeper server01:2181 –create –topic test –replication-factor 1 –partitions 3
    \n# \u5220\u9664topic \uff08\u5982\u679ckafka\u914d\u7f6edelete.topic.enable=true\uff0c\u90a3\u4e48\u53ef\u4ee5\u76f4\u63a5\u5220\u9664topic\uff0c\u6267\u884c\u5220\u9664topic\u547d\u4ee4\uff09
    \nkafka-topics.sh –zookeeper server01:2181 –delete –topic test
    \n# \u4fee\u6539topic\u7684\u5206\u533a\uff0c\u6ce8\u610f\uff1a\u5206\u533a\u6570\u91cf\u53ea\u80fd\u589e\u52a0\uff0c\u4e0d\u80fd\u51cf\u5c11
    \nkafka-topics.sh –zookeeper server01:2181 –alter –topic test –partitions 5
    \n# \u67e5\u770b\u6240\u6709topic
    \nkafka-topics.sh –zookeeper server01:2181 –list
    \n# \u67e5\u770b\u6240\u6709topic\u7684\u8be6\u7ec6\u4fe1\u606f
    \nkafka-topics.sh –zookeeper server01:2181 –describe
    \n# \u67e5\u770b\u6307\u5b9atopic\u7684\u8be6\u7ec6\u4fe1\u606f
    \nkafka-topics.sh –zookeeper server01:2181 –describe –topic test
    \n# describe\u547d\u4ee4\u8fd8\u63d0\u4f9b\u4e00\u4e9b\u53c2\u6570\uff0c\u7528\u4e8e\u8fc7\u6ee4\u8f93\u51fa\u7ed3\u679c\uff0c\u5982\uff1a
    \n# –topic-with-overrides\uff1a\u53ef\u4ee5\u627e\u51fa\u6240\u6709\u5305\u542b\u8986\u76d6\u914d\u7f6e\u7684\u4e3b\u9898\uff0c\u5b83\u53ea\u4f1a\u5217\u51fa\u5305\u542b\u4e0e\u96c6\u7fa4\u4e0d\u4e00\u6837\u914d\u7f6e\u7684\u4e3b\u9898
    \nkafka-topics.sh –zookeeper server01:2181 –describe –topics-with-overrides
    \n# describe\u6709\u4e24\u4e2a\u53c2\u6570\u7528\u4e8e\u627e\u51fa\u6709\u95ee\u9898\u7684\u5206\u533a
    \n# –unavailable-partitions\uff1a\u5217\u51fa\u6240\u6709\u6ca1\u6709\u9996\u9886\u7684\u5206\u533a\uff0c\u8fd9\u4e9b\u5206\u533a\u5df2\u7ecf\u5904\u4e8e\u79bb\u7ebf\u72b6\u6001\uff0c\u5bf9\u4e8e\u751f\u4ea7\u8005\u548c\u6d88\u8d39\u8005\u6765\u8bf4\u662f\u4e0d\u53ef\u7528\u7684
    \n# –under-replicated-partitions\uff1a\u5217\u51fa\u6240\u6709\u5305\u542b\u4e0d\u540c\u6b65\u526f\u672c\u7684\u5206\u533a\u3002
    \nkafka-topics.sh –zookeeper server01:2181 –describe –unavailable-partitions
    \nkafka-topics.sh –zookeeper server01:2181 –describe –under-replicated-partitions<\/p>\n
    \u751f\u4ea7\u8005shell\u547d\u4ee4
    \nkafka-console-producer.sh –broker-list server01:9092 –topic test
    \n\u6d88\u8d39\u8005shell\u547d\u4ee4(kafka-console-consumer.sh –bootstrap-server localhost:9092 –topic test –from-beginning\u65b0\u7248)
    \nkafka-console-consumer.sh –zookeeper server01:2181 –from-beginning –topic test
    \n# \u6307\u5b9agroup\u6d88\u8d39\u7ec4
    \nkafka-console-consumer.sh –zookeeper server01:2181 –from-beginning –group test_group –topic test<\/p>\n
    # \u5217\u51fatest_group\u6d88\u8d39\u7ec4\u7684\u8be6\u7ec6\u4fe1\u606f
    \nkafka-consumer-groups.sh –zookeeper\u00a0 server01:2181\u00a0\u00a0\u00a0 –describe\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0 –group \u00a0\u00a0 \u00a0test_group
    \ntest\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0 \u00a0\u00a0\u00a0 \u00a0\u00a0\u00a0 \u00a00\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0 \u00a0\u00a0\u00a0 \u00a08\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0 \u00a0\u00a0\u00a0 \u00a0\u00a0\u00a0 \u00a0 8\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0 0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0 –
    \ntest\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0 \u00a0\u00a0\u00a0 \u00a0\u00a0\u00a0 \u00a01\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0 \u00a0\u00a0\u00a0 \u00a08\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0 \u00a0\u00a0\u00a0 \u00a0 8\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0 0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0 –
    \ntest\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0 \u00a0\u00a0\u00a0 \u00a0\u00a0\u00a0 \u00a02\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0 \u00a0\u00a0\u00a0 \u00a08\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0 \u00a0\u00a0\u00a0 \u00a0 \u00a0\u00a0 \u00a0 8\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0 0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0 –
    \n# CURRENT-OFFSET: \u5f53\u524d\u6d88\u8d39\u8005\u7fa4\u7ec4\u6700\u8fd1\u63d0\u4ea4\u7684offset\uff0c\u4e5f\u5c31\u662f\u6d88\u8d39\u8005\u5206\u533a\u91cc\u8bfb\u53d6\u7684\u5f53\u524d\u4f4d\u7f6e
    \n# LOG-END-OFFSET: \u5f53\u524d\u6700\u9ad8\u6c34\u4f4d\u504f\u79fb\u91cf\uff0c\u4e5f\u5c31\u662f\u6700\u8fd1\u4e00\u4e2a\u8bfb\u53d6\u6d88\u606f\u7684\u504f\u79fb\u91cf\uff0c\u540c\u65f6\u4e5f\u662f\u6700\u8fd1\u4e00\u4e2a\u63d0\u4ea4\u5230\u96c6\u7fa4\u7684\u504f\u79fb\u91cf
    \n# LAG\uff1a\u6d88\u8d39\u8005\u7684CURRENT-OFFSET\u4e0ebroker\u7684LOG-END-OFFSET\u4e4b\u95f4\u7684\u5dee\u8ddd<\/p>\n
    # \u5220\u9664\u6d88\u8d39\u8005\u7fa4\u7ec4
    \nkafka-consumer-groups.sh –zookeeper server01:2181 –delete –group test_group
    \n# \u5220\u9664\u6d88\u8d39\u8005\u7fa4\u7ec4\u4e2d\u7684topic
    \nkafka-consumer-groups.sh –zookeeper server01:2181 –delete –group test_group –topic test<\/p>\n
     <\/p>\n","protected":false},"excerpt":{"rendered":"
    kafka\u6b63\u5e38\u8fd0\u884c\uff0c\u5fc5\u987b\u914d\u7f6ezookeeper\uff0c\u5426\u5219\u65e0\u8bba\u662fkafka\u96c6\u7fa4\u8fd8\u662f\u5ba2\u6237\u7aef\u7684\u751f\u5b58\u8005\u548c\u6d88\u8d39\u8005\u90fd\u65e0\u6cd5\u6b63\u5e38\u7684\u5de5\u4f5c\u7684\uff1b\u6240\u4ee5\u9700\u8981\u914d\u7f6e\u542f\u52a8zookeeper\u670d\u52a1\u3002 \uff081\uff09zookeeper\u9700\u8981java\u73af\u5883 # yum -y install java-1.8.0 \u89e3\u538b\u5230\u6307\u5b9a\u76ee\u5f55 tar -xzf kafka_2.13-2.8.0.tgz -C \/opt\/ \uff082\uff09\u8fd9\u91cckafka\u4e0b\u8f7d\u5305\u5df2\u7ecf\u5305\u62eczookeeper\u670d\u52a1\uff0c\u6240\u4ee5\u53ea\u9700\u4fee\u6539\u914d\u7f6e\u6587\u4ef6\uff0c\u542f\u52a8\u5373\u53ef\u3002 \u5982\u679c\u9700\u8981\u4e0b\u8f7d\u6307\u5b9azookeeper\u7248\u672c\uff1b\u53ef\u4ee5\u5355\u72ec\u53bbzookeeper\u5b98\u7f51http:\/\/mirrors.shu.edu.cn\/apache\/zookeeper\/\u4e0b\u8f7d\u6307\u5b9a\u7248\u672c\u3002 [root@~]# cd \/data\/kafka_2.13-2.8.0\/ [root@ kafka_2.11-2.1.0]# grep “^[^#]” config\/zookeeper.properties dataDir=\/tmp\/zookeeper\u00a0\u00a0\u00a0#\u6570\u636e\u5b58\u50a8\u76ee\u5f55 clientPort=2181\u00a0\u00a0\u00a0#zookeeper\u7aef\u53e3 maxClientCnxns=0 \u6ce8\uff1a\u53ef\u81ea\u884c\u6dfb\u52a0\u4fee\u6539zookeeper\u914d\u7f6e 3.3\u00a0\u914d\u7f6ekafka \uff081\uff09\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 [root@ kafka_2.13-2.8.0]# grep “^[^#]” config\/server.properties broker.id=0 listeners=PLAINTEXT:\/\/localhost:9092 num.network.threads=3 num.io.threads=8 socket.send.buffer.bytes=102400 socket.receive.buffer.bytes=102400 socket.request.max.bytes=104857600 log.dirs=\/tmp\/kafka-logs num.partitions=1 num.recovery.threads.per.data.dir=1 offsets.topic.replication.factor=1 transaction.state.log.replication.factor=1 transaction.state.log.min.isr=1 log.retention.hours=168 log.segment.bytes=1073741824 log.retention.check.interval.ms=300000 zookeeper.connect=localhost:2181 zookeeper.connection.timeout.ms=6000 group.initial.rebalance.delay.ms=0 \u6ce8\uff1a\u53ef\u6839\u636e\u81ea\u5df1\u9700\u6c42\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 \u00a0broker.id\uff1a\u552f\u4e00\u6807\u8bc6ID … Read more<\/a><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[16],"tags":[],"class_list":["post-406","post","type-post","status-publish","format-standard","hentry","category-kafka"],"_links":{"self":[{"href":"http:\/\/22pig.com\/index.php?rest_route=\/wp\/v2\/posts\/406","targetHints":{"allow":["GET"]}}],"collection":[{"href":"http:\/\/22pig.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/22pig.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/22pig.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/22pig.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=406"}],"version-history":[{"count":0,"href":"http:\/\/22pig.com\/index.php?rest_route=\/wp\/v2\/posts\/406\/revisions"}],"wp:attachment":[{"href":"http:\/\/22pig.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=406"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/22pig.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=406"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/22pig.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=406"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}