{"id":1983,"date":"2023-04-01T10:47:15","date_gmt":"2023-04-01T02:47:15","guid":{"rendered":"https:\/\/www.appblog.cn\/?p=1983"},"modified":"2023-04-22T07:44:14","modified_gmt":"2023-04-21T23:44:14","slug":"container-deployment-elk-7-10-suitable-for-production","status":"publish","type":"post","link":"https:\/\/www.appblog.cn\/index.php\/2023\/04\/01\/container-deployment-elk-7-10-suitable-for-production\/","title":{"rendered":"\u5bb9\u5668\u90e8\u7f72ELK7.10\uff0c\u9002\u7528\u4e8e\u751f\u4ea7"},"content":{"rendered":"<h2>ELK\u67b6\u6784\u7b80\u4ecb<\/h2>\n<p><img decoding=\"async\" src=\"http:\/\/www.yezhou.me\/AppBlog\/images\/\u8fd0\u7ef4\/ELK\u67b6\u6784.png\" alt=\"ELK\u67b6\u6784\" \/><\/p>\n<p><!-- more --><\/p>\n<ul>\n<li>\u9996\u5148<code>logstash<\/code>\u5177\u6709\u65e5\u5fd7\u91c7\u96c6\u3001\u8fc7\u6ee4\u3001\u7b5b\u9009\u7b49\u529f\u80fd\uff0c\u529f\u80fd\u5b8c\u5584\u4f46\u540c\u65f6\u4f53\u91cf\u4e5f\u4f1a\u6bd4\u8f83\u5927\uff0c\u6d88\u8017\u7cfb\u7edf\u8d44\u6e90\u81ea\u7136\u4e5f\u591a\u3002<code>filebeat<\/code>\u4f5c\u4e3a\u4e00\u4e2a\u8f7b\u91cf\u7ea7\u65e5\u5fd7\u91c7\u96c6\u5de5\u5177\uff0c\u867d\u7136\u6ca1\u6709\u8fc7\u6ee4\u7b5b\u9009\u529f\u80fd\uff0c\u4f46\u662f\u4ec5\u4ec5\u90e8\u7f72\u5728\u5e94\u7528\u670d\u52a1\u5668\u4f5c\u4e3a\u6211\u4eec\u91c7\u96c6\u65e5\u5fd7\u7684\u5de5\u5177\u53ef\u4ee5\u662f\u8bf4\u6700\u597d\u7684\u9009\u62e9\u3002\u4f46\u6211\u4eec\u6709\u4e9b\u65f6\u5019\u53ef\u80fd\u53c8\u9700\u8981logstash\u7684\u8fc7\u6ee4\u7b5b\u9009\u529f\u80fd\uff0c\u6240\u4ee5\u6211\u4eec\u5728\u91c7\u96c6\u65e5\u5fd7\u65f6\u7528filebeat\uff0c\u7136\u540e\u4ea4\u7ed9logstash\u8fc7\u6ee4\u7b5b\u9009\u3002<\/li>\n<li>\u5176\u6b21\uff0clogstash\u7684\u541e\u5410\u91cf\u662f\u6709\u9650\u7684\uff0c\u4e00\u65e6\u77ed\u65f6\u95f4\u5185filebeat\u4f20\u8fc7\u6765\u7684\u65e5\u5fd7\u8fc7\u591a\u4f1a\u4ea7\u751f\u5806\u79ef\u548c\u5835\u585e\uff0c\u5bf9\u65e5\u5fd7\u7684\u91c7\u96c6\u4e5f\u4f1a\u53d7\u5230\u5f71\u54cd\uff0c\u6240\u4ee5\u5728filebeat\u4e0elogstash\u4e2d\u95f4\u53c8\u52a0\u4e86\u4e00\u5c42<code>kafka<\/code>\u6d88\u606f\u961f\u5217\u6765\u7f13\u5b58\u6216\u8005\u8bf4\u89e3\u8026\uff0c\u5f53\u7136redis\u4e5f\u662f\u53ef\u4ee5\u7684\u3002\u8fd9\u6837\u5f53\u4f17\u591afilebeat\u8282\u70b9\u91c7\u96c6\u5927\u91cf\u65e5\u5fd7\u76f4\u63a5\u653e\u5230kafka\u4e2d\uff0clogstash\u6162\u6162\u7684\u8fdb\u884c\u6d88\u8d39\uff0c\u4e24\u8fb9\u4e92\u4e0d\u5e72\u6270\u3002<\/li>\n<li>\u81f3\u4e8e<code>zookeeper<\/code>\uff0c\u5206\u5e03\u5f0f\u670d\u52a1\u7ba1\u7406\u795e\u5668\uff0c\u76d1\u63a7\u7ba1\u7406kafka\u7684\u8282\u70b9\u6ce8\u518c\uff0c<code>topic<\/code>\u7ba1\u7406\u7b49\uff0c\u540c\u65f6\u5f25\u8865\u4e86kafka\u96c6\u7fa4\u8282\u70b9\u5bf9\u5916\u754c\u65e0\u6cd5\u611f\u77e5\u7684\u95ee\u9898\uff0ckafka\u5b9e\u9645\u5df2\u7ecf\u81ea\u5e26\u4e86zookeeper\uff0c\u8fd9\u91cc\u5c06\u4f1a\u4f7f\u7528\u72ec\u7acb\u7684zookeeper\u8fdb\u884c\u7ba1\u7406\uff0c\u65b9\u4fbf\u540e\u671fzookeeper\u96c6\u7fa4\u7684\u6269\u5c55\u3002<\/li>\n<\/ul>\n<h2>\u73af\u5883<\/h2>\n<ul>\n<li>\u963f\u91cc\u4e91ECS\uff1a5\u53f0\u90e8\u7f72ES\u8282\u70b9\uff0c3\u53f0\u5206\u522b\u90e8\u7f72logstash\u3001kafka\u3001zookeeper\u548ckibana\u7b49\u670d\u52a1<\/li>\n<li>\u963f\u91cc\u4e91ECS\u914d\u7f6e\uff1a5\u53f0 4\u683816G SSD\u78c1\u76d8\u30023\u53f0 4\u683816G SSD\u78c1\u76d8\u3002\u90fd\u662f Centos7.8\u7cfb\u7edf<\/li>\n<li>\u5b89\u88c5 docker \u548c docker-compose<\/li>\n<li>ELK\u7248\u672c7.10.1\uff1bzookeeper\u7248\u672c3.6.2\uff1bkafka\u7248\u672c2.13-2.6.0<\/li>\n<\/ul>\n<table>\n<thead>\n<tr>\n<th style=\"text-align: left;\">IP\u5730\u5740<\/th>\n<th style=\"text-align: left;\">\u4e3b\u673a\u540d\u79f0<\/th>\n<th style=\"text-align: left;\">\u7528\u9014<\/th>\n<\/tr>\n<\/thead>\n<tbody>\n<tr>\n<td style=\"text-align: left;\">172.20.166.25<\/td>\n<td style=\"text-align: left;\">es-master1<\/td>\n<td style=\"text-align: left;\">es master \u548c es \u6570\u636e\u8282\u70b9<\/td>\n<\/tr>\n<tr>\n<td style=\"text-align: left;\">172.20.166.24<\/td>\n<td style=\"text-align: left;\">es-master2<\/td>\n<td style=\"text-align: left;\">es master \u548c es \u6570\u636e\u8282\u70b9<\/td>\n<\/tr>\n<tr>\n<td style=\"text-align: left;\">172.20.166.22<\/td>\n<td style=\"text-align: left;\">es-master3<\/td>\n<td style=\"text-align: left;\">es master \u548c es \u6570\u636e\u8282\u70b9<\/td>\n<\/tr>\n<tr>\n<td style=\"text-align: left;\">172.20.166.23<\/td>\n<td style=\"text-align: left;\">es-data1<\/td>\n<td style=\"text-align: left;\">es\u6570\u636e\u8282\u70b9<\/td>\n<\/tr>\n<tr>\n<td style=\"text-align: left;\">172.20.166.26<\/td>\n<td style=\"text-align: left;\">es-data2<\/td>\n<td style=\"text-align: left;\">es\u6570\u636e\u8282\u70b9<\/td>\n<\/tr>\n<tr>\n<td style=\"text-align: left;\">172.20.166.27<\/td>\n<td style=\"text-align: left;\">logstash1<\/td>\n<td style=\"text-align: left;\">logstash\u3001kafka\u3001zookeeper<\/td>\n<\/tr>\n<tr>\n<td style=\"text-align: left;\">172.20.166.28<\/td>\n<td style=\"text-align: left;\">logstash2<\/td>\n<td style=\"text-align: left;\">logstash\u3001kafka\u3001zookeeper<\/td>\n<\/tr>\n<tr>\n<td style=\"text-align: left;\">172.20.166.29<\/td>\n<td style=\"text-align: left;\">logstash3<\/td>\n<td style=\"text-align: left;\">logstash\u3001kafka\u3001kafa-manager\u3001zookeeper\u3001kibana\u3001curator<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h2>\u7cfb\u7edf\u53c2\u6570\u4f18\u5316<\/h2>\n<pre><code class=\"language-bash\"># \u6700\u5927\u7528\u6237\u6253\u5f00\u8fdb\u7a0b\u6570\n$ vim \/etc\/security\/limits.d\/20-nproc.conf\n\n*           soft   nproc       65535\n*           hard   nproc       65535\n\n# \u4f18\u5316\u5185\u6838\uff0c\u7528\u4e8e docker \u652f\u6301\n$ modprobe br_netfilter\n$ cat &lt;&lt;EOF &gt;  \/etc\/sysctl.d\/k8s.conf\nnet.bridge.bridge-nf-call-ip6tables = 1\nnet.bridge.bridge-nf-call-iptables = 1\nnet.ipv4.ip_forward = 1\nEOF\n$ sysctl -p \/etc\/sysctl.d\/k8s.conf\n\n# \u4f18\u5316\u5185\u6838\uff0c\u5bf9 es \u652f\u6301\n$ echo &#039;vm.max_map_count=262144&#039; &gt;&gt; \/etc\/sysctl.conf\n\n# \u751f\u6548\u914d\u7f6e\n$ sysctl -p<\/code><\/pre>\n<h2>\u90e8\u7f72 docker \u548c docker-compose<\/h2>\n<h3>\u90e8\u7f72 docker<\/h3>\n<pre><code class=\"language-bash\"># \u5b89\u88c5\u5fc5\u8981\u7684\u4e00\u4e9b\u7cfb\u7edf\u5de5\u5177\n$ yum install -y yum-utils device-mapper-persistent-data lvm2\n\n# \u6dfb\u52a0\u8f6f\u4ef6\u6e90\u4fe1\u606f\n$ yum-config-manager --add-repo http:\/\/mirrors.aliyun.com\/docker-ce\/linux\/centos\/docker-ce.repo\n\n# \u66f4\u65b0\u5e76\u5b89\u88c5 Docker-CE\n$ yum makecache fast\n$ yum -y install docker-ce\n\n# \u914d\u7f6edocker\n$ systemctl enable docker\n$ systemctl start docker\n$ vim \/etc\/docker\/daemon.json\n{&quot;data-root&quot;: &quot;\/var\/lib\/docker&quot;, &quot;bip&quot;: &quot;10.50.0.1\/16&quot;, &quot;default-address-pools&quot;: [{&quot;base&quot;: &quot;10.51.0.1\/16&quot;, &quot;size&quot;: 24}], &quot;registry-mirrors&quot;: [&quot;https:\/\/4xr1qpsp.mirror.aliyuncs.com&quot;], &quot;log-opts&quot;: {&quot;max-size&quot;:&quot;500m&quot;, &quot;max-file&quot;:&quot;3&quot;}}\n$ sed  -i &#039;\/ExecStart=\/i ExecStartPost=\\\/sbin\\\/iptables -P FORWARD ACCEPT&#039; \/usr\/lib\/systemd\/system\/docker.service\n$ systemctl enable docker.service\n$ systemctl daemon-reload\n$ systemctl restart docker<\/code><\/pre>\n<h3>\u90e8\u7f72 docker-compose<\/h3>\n<pre><code class=\"language-bash\"># \u5b89\u88c5 docker-compose\n$ sudo curl -L &quot;https:\/\/github.com\/docker\/compose\/releases\/download\/1.27.4\/docker-compose-$(uname -s)-$(uname -m)&quot; -o \/usr\/local\/bin\/docker-compose\n$ chmod +x \/usr\/local\/bin\/docker-compose<\/code><\/pre>\n<h2>\u90e8\u7f72 ES<\/h2>\n<h3>es-master1 \u64cd\u4f5c<\/h3>\n<pre><code class=\"language-bash\"># \u521b\u5efa es \u76ee\u5f55\n$ mkdir \/data\/ELKStack\n$ mkdir elasticsearch elasticsearch-data elasticsearch-plugins\n\n# \u5bb9\u5668es\u7528\u6237 uid \u548c gid \u90fd\u662f 1000\n$ chown 1000.1000 elasticsearch-data elasticsearch-plugins\n\n# \u4e34\u65f6\u542f\u52a8\u4e00\u4e2aes\n$ docker run --name es-test -it --rm docker.elastic.co\/elasticsearch\/elasticsearch:7.10.1 bash\n\n# \u751f\u6210\u8bc1\u4e66\uff0c\u8bc1\u4e66\u6709\u6548\u671f10\u5e74\uff0c\u8bc1\u4e66\u8f93\u5165\u7684\u5bc6\u7801\u8fd9\u91cc\u4e3a\u7a7a\n$ bin\/elasticsearch-certutil ca --days 3660\n$ bin\/elasticsearch-certutil cert --ca elastic-stack-ca.p12 --days 3660\n\n# \u6253\u5f00\u65b0\u7684\u7a97\u53e3\uff0c\u62f7\u8d1d\u751f\u6210\u7684\u8bc1\u4e66\n$ cd \/data\/ELKStack\/elasticsearch\n$ mkdir es-p12\n$ docker cp es-test:\/usr\/share\/elasticsearch\/elastic-certificates.p12 .\/es-p12\n$ docker cp es-test:\/usr\/share\/elasticsearch\/elastic-stack-ca.p12 .\/es-p12\n$ chown -R 1000.1000 .\/es-p12\n\n# \u521b\u5efa docker-compose.yml\n$ vim docker-compose.yml\n\nversion: &#039;2.2&#039;\nservices:\n  elasticsearch:\n    image: docker.elastic.co\/elasticsearch\/elasticsearch:7.10.1\n    container_name: es01\n    environment:\n      - cluster.name=es-docker-cluster\n      - cluster.initial_master_nodes=es01,es02,es03\n      - bootstrap.memory_lock=true\n      - &quot;ES_JAVA_OPTS=-Xms10000m -Xmx10000m&quot;\n    ulimits:\n      memlock:\n        soft: -1\n        hard: -1\n      nofile:\n        soft: 65536\n        hard: 65536\n    mem_limit: 13000m\n    cap_add:\n      - IPC_LOCK\n    restart: always\n    # \u8bbe\u7f6e docker host \u7f51\u7edc\u6a21\u5f0f\n    network_mode: &quot;host&quot;\n    volumes:\n       - \/data\/ELKStack\/elasticsearch-data:\/usr\/share\/elasticsearch\/data\n       - \/data\/ELKStack\/elasticsearch-plugins:\/usr\/share\/elasticsearch\/plugins\n       - \/data\/ELKStack\/elasticsearch\/elasticsearch.yml:\/usr\/share\/elasticsearch\/config\/elasticsearch.yml\n       - \/data\/ELKStack\/elasticsearch\/es-p12:\/usr\/share\/elasticsearch\/config\/es-p12\n\n# \u521b\u5efa elasticsearch.yml \u914d\u7f6e\u6587\u4ef6\n$ vim elasticsearch.yml\n\ncluster.name: &quot;es-docker-cluster&quot;\nnode.name: &quot;es01&quot;\nnetwork.host: 0.0.0.0\n\nnode.master: true\nnode.data: true\n\ndiscovery.zen.minimum_master_nodes: 2\nhttp.port: 9200\ntransport.tcp.port: 9300\n\n# \u5982\u679c\u662f\u591a\u8282\u70b9es\uff0c\u901a\u8fc7ping\u6765\u5065\u5eb7\u68c0\u67e5\ndiscovery.zen.ping.unicast.hosts: [&quot;172.20.166.25:9300&quot;, &quot;172.20.166.24:9300&quot;, &quot;172.20.166.22:9300&quot;, &quot;172.20.166.23:9300&quot;, &quot;172.20.166.26:9300&quot;]\ndiscovery.zen.fd.ping_timeout: 120s\ndiscovery.zen.fd.ping_retries: 6\ndiscovery.zen.fd.ping_interval: 10s\n\ncluster.info.update.interval: 1m\nindices.fielddata.cache.size:  20%\nindices.breaker.fielddata.limit: 40%\nindices.breaker.request.limit: 40%\nindices.breaker.total.limit: 70%\nindices.memory.index_buffer_size: 20%\nscript.painless.regex.enabled: true\n\n# \u78c1\u76d8\u5206\u7247\u5206\u914d\ncluster.routing.allocation.disk.watermark.low: 100gb\ncluster.routing.allocation.disk.watermark.high: 50gb\ncluster.routing.allocation.disk.watermark.flood_stage: 30gb\n\n# \u672c\u5730\u6570\u636e\u5206\u7247\u6062\u590d\u914d\u7f6e\ngateway.recover_after_nodes: 3\ngateway.recover_after_time: 5m\ngateway.expected_nodes: 3\ncluster.routing.allocation.node_initial_primaries_recoveries: 8\ncluster.routing.allocation.node_concurrent_recoveries: 2\n\n# \u5141\u8bb8\u8de8\u57df\u8bf7\u6c42\nhttp.cors.enabled: true\nhttp.cors.allow-origin: &quot;*&quot;\nhttp.cors.allow-headers: Authorization,X-Requested-With,Content-Length,Content-Type\n\n# \u5f00\u542fxpack\nxpack.security.enabled: true\nxpack.monitoring.collection.enabled: true\n\n# \u5f00\u542f\u96c6\u7fa4\u4e2dhttps\u4f20\u8f93\nxpack.security.transport.ssl.enabled: true\nxpack.security.transport.ssl.verification_mode: certificate\nxpack.security.transport.ssl.keystore.path: es-p12\/elastic-certificates.p12\nxpack.security.transport.ssl.truststore.path: es-p12\/elastic-certificates.p12\n\n# \u628a es \u914d\u7f6e\u4f7f\u7528 rsync \u540c\u6b65\u5230\u5176\u5b83 es \u8282\u70b9\n$ rsync -avp -e ssh \/data\/ELKStack 172.20.166.24:\/data\/\n$ rsync -avp -e ssh \/data\/ELKStack 172.20.166.22:\/data\/\n$ rsync -avp -e ssh \/data\/ELKStack 172.20.166.23:\/data\/\n$ rsync -avp -e ssh \/data\/ELKStack 172.20.166.26:\/data\/\n\n# \u542f\u52a8 es\n$ docker-compose up -d\n\n# \u67e5\u770b es\n$ docker-compose ps<\/code><\/pre>\n<h3>es-master2 \u64cd\u4f5c<\/h3>\n<pre><code class=\"language-bash\">$ cd \/data\/ELKStack\/elasticsearch\n\n# \u4fee\u6539 docker-compose.yml elasticsearch.yml \u4e24\u4e2a\u914d\u7f6e\n$ sed -i &#039;s\/es01\/es02\/g&#039; docker-compose.yml elasticsearch.yml\n\n# \u542f\u52a8 es\n$ docker-compose up -d<\/code><\/pre>\n<h3>es-master3 \u64cd\u4f5c<\/h3>\n<pre><code class=\"language-bash\">$ cd \/data\/ELKStack\/elasticsearch\n\n# \u4fee\u6539 docker-compose.yml elasticsearch.yml \u4e24\u4e2a\u914d\u7f6e\n$ sed -i &#039;s\/es01\/es03\/g&#039; docker-compose.yml elasticsearch.yml\n\n# \u542f\u52a8 es\n$ docker-compose up -d<\/code><\/pre>\n<h3>es-data1 \u64cd\u4f5c<\/h3>\n<pre><code class=\"language-bash\">$ cd \/data\/ELKStack\/elasticsearch\n\n# \u4fee\u6539 docker-compose.yml elasticsearch.yml \u4e24\u4e2a\u914d\u7f6e\n$ sed -i &#039;s\/es01\/es04\/g&#039; docker-compose.yml elasticsearch.yml\n\n# \u4e0d\u505a\u4e3a es master \u8282\u70b9\uff0c\u53ea\u505a\u6570\u636e\u8282\u70b9\n$ sed -i &#039;s\/node.master: true\/node.master: false\/g&#039; elasticsearch.yml\n\n# \u542f\u52a8 es\n$ docker-compose up -d<\/code><\/pre>\n<h3>es-data2 \u64cd\u4f5c<\/h3>\n<pre><code class=\"language-bash\">$ cd \/data\/ELKStack\/elasticsearch\n\n# \u4fee\u6539 docker-compose.yml elasticsearch.yml \u4e24\u4e2a\u914d\u7f6e\n$ sed -i &#039;s\/es01\/es05\/g&#039; docker-compose.yml elasticsearch.yml\n\n# \u4e0d\u505a\u4e3a es master \u8282\u70b9\uff0c\u53ea\u505a\u6570\u636e\u8282\u70b9\n$ sed -i &#039;s\/node.master: true\/node.master: false\/g&#039; elasticsearch.yml\n\n# \u542f\u52a8 es\n$ docker-compose up -d<\/code><\/pre>\n<h3>\u8bbe\u7f6e es \u8bbf\u95ee\u8d26\u53f7<\/h3>\n<pre><code class=\"language-bash\"># es-master1 \u64cd\u4f5c\n$ docker exec -it es01 bash\n\n# \u8bbe\u7f6e elastic\uff0capm_system\uff0ckibana\uff0ckibana_system\uff0clogstash_system\uff0cbeats_system\uff0cremote_monitoring_user \u7b49\u5bc6\u7801\n# \u5bc6\u7801\u90fd\u8bbe\u7f6e\u4e3a elastic123\uff0c\u8fd9\u91cc\u53ea\u662f\u4e3e\u4f8b\uff0c\u5177\u4f53\u6839\u636e\u9700\u6c42\u8bbe\u7f6e\n$ .\/bin\/elasticsearch-setup-passwords interactive<\/code><\/pre>\n<h2>\u90e8\u7f72 Kibana<\/h2>\n<h3>logstash3 \u64cd\u4f5c<\/h3>\n<pre><code class=\"language-bash\">$ mkdir -p \/data\/ELKStack\/kibana\n$ cd \/data\/ELKStack\/kibana\n\n# \u521b\u5efa kibana \u76f8\u5173\u76ee\u5f55\uff0c\u7528\u4e8e\u5bb9\u5668\u6302\u8f7d\n$ mkdir config data plugins\n$ chown 1000.1000 config data plugins\n\n# \u521b\u5efa docker-compose.yml\n$ vim docker-compose.yml\n\nversion: &#039;2&#039;\nservices:\n  kibana:\n    image: docker.elastic.co\/kibana\/kibana:7.10.1\n    container_name: kibana\n    restart: always\n    network_mode: &quot;bridge&quot;\n    mem_limit: 2000m\n    environment:\n      SERVER_NAME: kibana.example.com\n    ports:\n      - &quot;5601:5601&quot;\n    volumes:\n       - \/data\/ELKStack\/kibana\/config:\/usr\/share\/kibana\/config\n       - \/data\/ELKStack\/kibana\/data:\/usr\/share\/kibana\/data\n       - \/data\/ELKStack\/kibana\/plugins:\/usr\/share\/kibana\/plugins\n\n# \u521b\u5efa kibana.yml\n$ vim config\/kibana.yml\n\nserver.name: kibana\nserver.host: &quot;0&quot;\nelasticsearch.hosts: [&quot;http:\/\/172.20.166.25:9200&quot;,&quot;http:\/\/172.20.166.24:9200&quot;,&quot;http:\/\/172.20.166.22:9200&quot;]\nelasticsearch.username: &quot;kibana&quot;\nelasticsearch.password: &quot;elastic123&quot;\nmonitoring.ui.container.elasticsearch.enabled: true\nxpack.security.enabled: true\nxpack.encryptedSavedObjects.encryptionKey: encryptedSavedObjects12345678909876543210\nxpack.security.encryptionKey: encryptionKeysecurity12345678909876543210\nxpack.reporting.encryptionKey: encryptionKeyreporting12345678909876543210\ni18n.locale: &quot;zh-CN&quot;\n\n# \u542f\u52a8 kibana\n$ docker-compose up -d<\/code><\/pre>\n<h2>\u90e8\u7f72 Zookeeper<\/h2>\n<h3>logstash1 \u64cd\u4f5c<\/h3>\n<pre><code class=\"language-bash\"># \u521b\u5efa zookeeper \u76ee\u5f55\n$ mkdir \/data\/ELKStack\/zookeeper\n$ cd \/data\/ELKStack\/zookeeper\n$ mkdir data datalog\n$ chown 1000.1000 data datalog\n\n# \u521b\u5efa docker-compose.yml\n$ vim docker-compose.yml\n\nversion: &#039;2&#039;\nservices:\n  zoo1:\n    image: zookeeper:3.6.2\n    restart: always\n    hostname: zoo1\n    container_name: zoo1\n    network_mode: &quot;bridge&quot;\n    mem_limit: 2000m\n    ports:\n      - 2181:2181\n      - 3888:3888\n      - 2888:2888\n    volumes:\n      - \/data\/ELKStack\/zookeeper\/data:\/data\n      - \/data\/ELKStack\/zookeeper\/datalog:\/datalog\n      - \/data\/ELKStack\/zookeeper\/zoo.cfg:\/conf\/zoo.cfg\n    environment:\n      ZOO_MY_ID: 1  # \u8868\u793a ZK\u670d\u52a1\u7684 id, \u5b83\u662f1-255 \u4e4b\u95f4\u7684\u6574\u6570, \u5fc5\u987b\u5728\u96c6\u7fa4\u4e2d\u552f\u4e00\n      ZOO_SERVERS: server.1=0.0.0.0:2888:3888;2181 server.2=172.20.166.28:2888:3888;2181 server.3=172.20.166.29:2888:3888;2181\n      # ZOOKEEPER_CLIENT_PORT: 2181\n\n# \u521b\u5efa zoo.cfg \u914d\u7f6e\n$ vim zoo.cfg\n\ntickTime=2000\ninitLimit=10\nsyncLimit=5\ndataDir=\/data\ndataLogDir=\/datalog\nautopurge.snapRetainCount=3\nautopurge.purgeInterval=1\nmaxClientCnxns=60\nserver.1= 0.0.0.0:2888:3888;2181\nserver.2= 172.20.166.28:2888:3888;2181\nserver.3= 172.20.166.29:2888:3888;2181\n\n# \u62f7\u8d1d\u914d\u7f6e\u5230 logstash2 logstash3 \u673a\u5668\u4e0a\n$ rsync -avp -e ssh \/data\/ELKStack\/zookeeper 172.20.166.28:\/data\/ELKStack\/\n$ rsync -avp -e ssh \/data\/ELKStack\/zookeeper 172.20.166.29:\/data\/ELKStack\/\n\n# \u542f\u52a8 zookeeper\n$ docker-compose up -d<\/code><\/pre>\n<h3>logstash2 \u64cd\u4f5c<\/h3>\n<pre><code class=\"language-bash\">$ cd \/data\/ELKStack\/zookeeper\n\n# \u4fee\u6539 docker-compose.yml \u6587\u4ef6\n$ vim docker-compose.yml\n\nversion: &#039;2&#039;\nservices:\n  zoo2:\n    image: zookeeper:3.6.2\n    restart: always\n    hostname: zoo2\n    container_name: zoo2\n    network_mode: &quot;bridge&quot;\n    mem_limit: 2000m\n    ports:\n      - 2181:2181\n      - 3888:3888\n      - 2888:2888\n    volumes:\n      - \/data\/ELKStack\/zookeeper\/data:\/data\n      - \/data\/ELKStack\/zookeeper\/datalog:\/datalog\n      - \/data\/ELKStack\/zookeeper\/zoo.cfg:\/conf\/zoo.cfg\n    environment:\n      ZOO_MY_ID: 2  # \u8868\u793a ZK\u670d\u52a1\u7684 id, \u5b83\u662f1-255 \u4e4b\u95f4\u7684\u6574\u6570, \u5fc5\u987b\u5728\u96c6\u7fa4\u4e2d\u552f\u4e00\n      ZOO_SERVERS: server.1=172.20.166.27:2888:3888;2181 server.2=0.0.0.0:2888:3888;2181 server.3=172.20.166.29:2888:3888;2181\n      # ZOOKEEPER_CLIENT_PORT: 2181\n\n# \u4fee\u6539 zoo.cfg\n$ vim zoo.cfg\n\ntickTime=2000\ninitLimit=10\nsyncLimit=5\ndataDir=\/data\ndataLogDir=\/datalog\nautopurge.snapRetainCount=3\nautopurge.purgeInterval=1\nmaxClientCnxns=60\nserver.1= 172.20.166.27:2888:3888;2181\nserver.2= 0.0.0.0:2888:3888;2181\nserver.3= 172.20.166.29:2888:3888;2181\n\n# \u542f\u52a8 zookeeper\n$ docker-compose up -d<\/code><\/pre>\n<h3>logstash3 \u64cd\u4f5c<\/h3>\n<pre><code class=\"language-bash\">$ cd \/data\/ELKStack\/zookeeper\n\n# \u4fee\u6539 docker-compose.yml \u6587\u4ef6\n$ vim docker-compose.yml\n\nversion: &#039;2&#039;\nservices:\n  zoo3:\n    image: zookeeper:3.6.2\n    restart: always\n    hostname: zoo3\n    container_name: zoo3\n    network_mode: &quot;bridge&quot;\n    mem_limit: 2000m\n    ports:\n      - 2181:2181\n      - 3888:3888\n      - 2888:2888\n    volumes:\n      - \/data\/ELKStack\/zookeeper\/data:\/data\n      - \/data\/ELKStack\/zookeeper\/datalog:\/datalog\n      - \/data\/ELKStack\/zookeeper\/zoo.cfg:\/conf\/zoo.cfg\n    environment:\n      ZOO_MY_ID: 3  # \u8868\u793a ZK\u670d\u52a1\u7684 id, \u5b83\u662f1-255 \u4e4b\u95f4\u7684\u6574\u6570, \u5fc5\u987b\u5728\u96c6\u7fa4\u4e2d\u552f\u4e00\n      ZOO_SERVERS: server.1=172.20.166.27:2888:3888;2181 server.2=172.20.166.28:2888:3888;2181 server.3=0.0.0.0:2888:3888;2181\n      # ZOOKEEPER_CLIENT_PORT: 2181\n\n# \u4fee\u6539 zoo.cfg\n$ vim zoo.cfg\n\ntickTime=2000\ninitLimit=10\nsyncLimit=5\ndataDir=\/data\ndataLogDir=\/datalog\nautopurge.snapRetainCount=3\nautopurge.purgeInterval=1\nmaxClientCnxns=60\nserver.1= 172.20.166.27:2888:3888;2181\nserver.2= 172.20.166.28:2888:3888;2181\nserver.3= 0.0.0.0:2888:3888;2181\n\n# \u542f\u52a8 zookeeper\n$ docker-compose up -d\n\n# \u64cd\u4f5c zookeeper\n$ docker run -it zoo3 bash\n$ zkCli.sh -server 172.20.166.27:2181,172.20.166.28:2181,172.20.166.29:2181<\/code><\/pre>\n<h2>\u90e8\u7f72 Kafka<\/h2>\n<h3>logstash1 \u64cd\u4f5c<\/h3>\n<pre><code class=\"language-bash\"># \u521b\u5efa kafka \u76ee\u5f55\n$ mkdir -p \/data\/ELKStack\/kafka\n$ cd \/data\/ELKStack\/kafka\n\n# \u521b\u5efa\u6570\u636e\u76ee\u5f55\uff0c\u7528\u4e8e\u5b58\u50a8kafka\u5bb9\u5668\u6570\u636e\n$ mkdir data\n\n# \u628akafka\u914d\u7f6e\u62f7\u8d1d\u5230\u5bbf\u4e3b\u673a\u4e0a\n$ docker run --name kafka-test -it --rm wurstmeister\/kafka:2.13-2.6.0 bash\n$ cd \/opt\/kafka\n$ tar zcvf \/tmp\/config.tar.gz config\n\n# \u6253\u5f00\u4e00\u4e2a\u65b0\u7684\u7a97\u53e3\n$ docker cp kafka-test:\/tmp\/config.tar.gz .\/\n\n# \u89e3\u538b\u914d\u7f6e\u6587\u4ef6\n$ tar xf config.tar.gz\n\n# \u521b\u5efa docker-compose.yml\n$ vim docker-compose.yml\n\nversion: &#039;2&#039;\n\nservices:\n  kafka1:\n    image: wurstmeister\/kafka:2.13-2.6.0\n    restart: always\n    hostname: kafka1\n    container_name: kafka1\n    network_mode: &quot;bridge&quot;\n    mem_limit: 5120m\n    ports:\n    - 9092:9092\n    - 9966:9966\n    environment:\n      KAFKA_BROKER_ID: 1\n      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT:\/\/172.20.166.27:9092       # \u5bbf\u4e3b\u673a\u7684IP\u5730\u5740\u800c\u975e\u5bb9\u5668\u7684IP\uff0c\u53ca\u66b4\u9732\u51fa\u6765\u7684\u7aef\u53e3\n      KAFKA_ADVERTISED_HOST_NAME: 172.20.166.27                        # \u5916\u7f51\u8bbf\u95ee\u5730\u5740\n      KAFKA_ADVERTISED_PORT: 9092                                      # \u7aef\u53e3\n      KAFKA_ZOOKEEPER_CONNECT: 172.20.166.27:2181,172.20.166.28:2181,172.20.166.29:2181           # \u8fde\u63a5\u7684zookeeper\u670d\u52a1\u53ca\u7aef\u53e3\n      KAFKA_JMX_OPTS: &quot;-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=172.20.166.27 -Dcom.sun.management.jmxremote.rmi.port=9966&quot;\n      JMX_PORT: 9966 # kafka\u9700\u8981\u76d1\u63a7broker\u548ctopic\u7684\u6570\u636e\u7684\u65f6\u5019,\u662f\u9700\u8981\u5f00\u542fjmx_port\u7684\n      KAFKA_HEAP_OPTS: &quot;-Xmx4096M -Xms4096M&quot;\n    volumes:\n    - \/data\/ELKStack\/kafka\/data:\/kafka                    # kafka\u6570\u636e\u6587\u4ef6\u5b58\u50a8\u76ee\u5f55\n    - \/data\/ELKStack\/kafka\/config:\/opt\/kafka\/config\n\n# \u4f18\u5316 kafka server.properties \u914d\u7f6e\n$ vim config\/server.properties\n\n# \u8c03\u5927socket\uff0c\u9632\u6b62\u62a5\u9519\nsocket.send.buffer.bytes=1024000\nsocket.receive.buffer.bytes=1024000\nsocket.request.max.bytes=1048576000\n\n# topic \u6570\u636e\u4fdd\u7559\u591a\u4e45\uff0c\u9ed8\u8ba4168\u5c0f\u65f6(7day)\nlog.retention.hours=72\nlog.cleanup.policy=delete\n\n# \u62f7\u8d1d\u914d\u7f6e\u5230 logstash2 logstash3 \u673a\u5668\u4e0a\n$ rsync -avp -e ssh \/data\/ELKStack\/kafka 172.20.166.28:\/data\/ELKStack\/\n$ rsync -avp -e ssh \/data\/ELKStack\/kafka 172.20.166.29:\/data\/ELKStack\/\n\n# \u542f\u52a8 kafka\n$ docker-compose up -d<\/code><\/pre>\n<h3>logstash2 \u64cd\u4f5c<\/h3>\n<pre><code class=\"language-bash\">$ cd \/data\/ELKStack\/kafka\n\n# \u4fee\u6539 docker-compose.yml \u6587\u4ef6\n$ vim docker-compose.yml\n\nversion: &#039;2&#039;\n\nservices:\n  kafka2:\n    image: wurstmeister\/kafka:2.13-2.6.0\n    restart: always\n    hostname: kafka2\n    container_name: kafka2\n    network_mode: &quot;bridge&quot;\n    mem_limit: 5120m\n    ports:\n    - 9092:9092\n    - 9966:9966\n    environment:\n      KAFKA_BROKER_ID: 2\n      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT:\/\/172.20.166.28:9092       # \u5bbf\u4e3b\u673a\u7684IP\u5730\u5740\u800c\u975e\u5bb9\u5668\u7684IP\uff0c\u53ca\u66b4\u9732\u51fa\u6765\u7684\u7aef\u53e3\n      KAFKA_ADVERTISED_HOST_NAME: 172.20.166.28                        # \u5916\u7f51\u8bbf\u95ee\u5730\u5740\n      KAFKA_ADVERTISED_PORT: 9092                                      # \u7aef\u53e3\n      KAFKA_ZOOKEEPER_CONNECT: 172.20.166.27:2181,172.20.166.28:2181,172.20.166.29:2181           # \u8fde\u63a5\u7684zookeeper\u670d\u52a1\u53ca\u7aef\u53e3\n      KAFKA_JMX_OPTS: &quot;-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=172.20.166.28 -Dcom.sun.management.jmxremote.rmi.port=9966&quot;\n      JMX_PORT: 9966  # kafka\u9700\u8981\u76d1\u63a7broker\u548ctopic\u7684\u6570\u636e\u7684\u65f6\u5019,\u662f\u9700\u8981\u5f00\u542fjmx_port\u7684\n      KAFKA_HEAP_OPTS: &quot;-Xmx4096M -Xms4096M&quot;\n    volumes:\n    - \/data\/ELKStack\/kafka\/data:\/kafka                    # kafka\u6570\u636e\u6587\u4ef6\u5b58\u50a8\u76ee\u5f55\n    - \/data\/ELKStack\/kafka\/config:\/opt\/kafka\/config\n\n# \u542f\u52a8 kafka\n$ docker-compose up -d<\/code><\/pre>\n<h3>logstash3 \u64cd\u4f5c<\/h3>\n<pre><code>$ cd \/data\/ELKStack\/kafka\n\n# \u4fee\u6539 docker-compose.yml \u6587\u4ef6\n$ vim docker-compose.yml\n\nversion: &#039;2&#039;\n\nservices:\n  kafka3:\n    image: wurstmeister\/kafka:2.13-2.6.0\n    restart: always\n    hostname: kafka3\n    container_name: kafka3\n    network_mode: &quot;bridge&quot;\n    mem_limit: 5120m\n    ports:\n    - 9092:9092\n    - 9966:9966\n    environment:\n      KAFKA_BROKER_ID: 3\n      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT:\/\/172.20.166.29:9092       # \u5bbf\u4e3b\u673a\u7684IP\u5730\u5740\u800c\u975e\u5bb9\u5668\u7684IP\uff0c\u53ca\u66b4\u9732\u51fa\u6765\u7684\u7aef\u53e3\n      KAFKA_ADVERTISED_HOST_NAME: 172.20.166.29                        # \u5916\u7f51\u8bbf\u95ee\u5730\u5740\n      KAFKA_ADVERTISED_PORT: 9092                                      # \u7aef\u53e3\n      KAFKA_ZOOKEEPER_CONNECT: 172.20.166.27:2181,172.20.166.28:2181,172.20.166.29:2181           # \u8fde\u63a5\u7684zookeeper\u670d\u52a1\u53ca\u7aef\u53e3\n      KAFKA_JMX_OPTS: &quot;-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=172.20.166.29 -Dcom.sun.management.jmxremote.rmi.port=9966&quot;\n      JMX_PORT: 9966  # kafka\u9700\u8981\u76d1\u63a7broker\u548ctopic\u7684\u6570\u636e\u7684\u65f6\u5019,\u662f\u9700\u8981\u5f00\u542fjmx_port\u7684\n      KAFKA_HEAP_OPTS: &quot;-Xmx4096M -Xms4096M&quot;\n    volumes:\n    - \/data\/ELKStack\/kafka\/data:\/kafka                    # kafka\u6570\u636e\u6587\u4ef6\u5b58\u50a8\u76ee\u5f55\n    - \/data\/ELKStack\/kafka\/config:\/opt\/kafka\/config\n\n# \u542f\u52a8 kafka\n$ docker-compose up -d\n\n# \u90e8\u7f72 kafka-manager \u7ba1\u7406 kafka \u5e73\u53f0\n$ mkdir \/data\/ELKStack\/kafka-manager\n$ cd \/data\/ELKStack\/kafka-manager\n$ vim docker-compose.yml\n\nversion: &#039;3.6&#039;\nservices:\n  kafka_manager:\n    restart: always\n    container_name: kafa-manager\n    hostname: kafka-manager\n    network_mode: &quot;bridge&quot;\n    mem_limit: 1024m\n    image: hlebalbau\/kafka-manager:3.0.0.5-7e7a22e\n    ports:\n      - &quot;9000:9000&quot;\n    environment:\n      ZK_HOSTS: &quot;172.20.166.27:2181,172.20.166.28:2181,172.20.166.29:2181&quot;\n      APPLICATION_SECRET: &quot;random-secret&quot;\n      KAFKA_MANAGER_AUTH_ENABLED: &quot;true&quot;\n      KAFKA_MANAGER_USERNAME: admin\n      KAFKA_MANAGER_PASSWORD: elastic123\n      JMX_PORT: 9966\n      TZ: &quot;Asia\/Shanghai&quot;\n\n# \u542f\u52a8 kafka-manager\n$ docker-compose up -d\n\n# \u8bbf\u95ee http:\/\/172.20.166.29:9000 \uff0c\u628a\u4e0a\u9762\u521b\u5efa\u7684\u4e09\u53f0 kafka \u52a0\u5165\u7ba1\u7406\uff0c\u8fd9\u91cc\u4e0d\u5728\u9610\u8ff0\uff0c\u7f51\u4e0a\u5f88\u591a\u914d\u7f6e\u6559\u7a0b<\/code><\/pre>\n<h2>\u90e8\u7f72 logstash<\/h2>\n<h3>logstash1 \u64cd\u4f5c<\/h3>\n<pre><code class=\"language-bash\">$ mkdir \/data\/ELKStack\/logstash\n$ cd \/data\/ELKStack\/logstash\n$ mkdir config data\n$ chown 1000.1000 config data\n\n# \u521b\u5efa docker-compose.yml\n$ vim docker-compose.yml\n\nversion: &#039;2&#039;\nservices:\n  logstash1:\n    image: docker.elastic.co\/logstash\/logstash:7.10.1\n    container_name: logstash1\n    hostname: logstash1\n    restart: always\n    network_mode: &quot;bridge&quot;\n    mem_limit: 4096m\n    environment:\n      TZ: &quot;Asia\/Shanghai&quot;\n    ports:\n      - 5044:5044\n    volumes:\n      - \/data\/ELKStack\/logstash\/config:\/config-dir\n      - \/data\/ELKStack\/logstash\/logstash.yml:\/usr\/share\/logstash\/config\/logstash.yml\n      - \/data\/ELKStack\/logstash\/data:\/usr\/share\/logstash\/data\n      - \/etc\/localtime:\/etc\/localtime\n    user: logstash\n    command: bash -c &quot;logstash -f \/config-dir --config.reload.automatic&quot;\n\n# \u521b\u5efa logstash.yml\n$ vim logstash.yml\n\nhttp.host: &quot;0.0.0.0&quot;\n# \u6307\u53d1\u9001\u5230Elasticsearch\u7684\u6279\u91cf\u8bf7\u6c42\u7684\u5927\u5c0f\uff0c\u503c\u8d8a\u5927\uff0c\u5904\u7406\u5219\u901a\u5e38\u66f4\u9ad8\u6548\uff0c\u4f46\u589e\u52a0\u4e86\u5185\u5b58\u5f00\u9500\npipeline.batch.size: 3000\n# \u6307\u8c03\u6574Logstash\u7ba1\u9053\u7684\u5ef6\u8fdf\uff0c\u8fc7\u4e86\u8be5\u65f6\u95f4\u5219logstash\u5f00\u59cb\u6267\u884c\u8fc7\u6ee4\u5668\u548c\u8f93\u51fa\npipeline.batch.delay: 200\n\n# \u521b\u5efa logstash \u89c4\u5219\u914d\u7f6e\n$ vim config\/01-input.conf\n\ninput {                                        # \u8f93\u5165\u7ec4\u4ef6\n    kafka {                                    # \u4ecekafka\u6d88\u8d39\u6570\u636e\n        bootstrap_servers =&gt; [&quot;172.20.166.27:9092,172.20.166.28:9092,172.20.166.29:9092&quot;]\n        #topics =&gt; &quot;%{[@metadata][topic]}&quot;     # \u4f7f\u7528kafka\u4f20\u8fc7\u6765\u7684topic\n        topics_pattern =&gt; &quot;elk-.*&quot;             # \u4f7f\u7528\u6b63\u5219\u5339\u914dtopic\n        codec =&gt; &quot;json&quot;                        # \u6570\u636e\u683c\u5f0f\n        consumer_threads =&gt; 3                  # \u6d88\u8d39\u7ebf\u7a0b\u6570\u91cf\n        decorate_events =&gt; true                # \u53ef\u5411\u4e8b\u4ef6\u6dfb\u52a0Kafka\u5143\u6570\u636e\uff0c\u6bd4\u5982\u4e3b\u9898\u3001\u6d88\u606f\u5927\u5c0f\u7684\u9009\u9879\uff0c\u8fd9\u5c06\u5411logstash\u4e8b\u4ef6\u4e2d\u6dfb\u52a0\u4e00\u4e2a\u540d\u4e3akafka\u7684\u5b57\u6bb5\n        auto_offset_reset =&gt; &quot;latest&quot;          # \u81ea\u52a8\u91cd\u7f6e\u504f\u79fb\u91cf\u5230\u6700\u65b0\u7684\u504f\u79fb\u91cf\n        group_id =&gt; &quot;logstash-node&quot;            # \u6d88\u8d39\u7ec4ID\uff0c\u591a\u4e2a\u6709\u76f8\u540cgroup_id\u7684logstash\u5b9e\u4f8b\u4e3a\u4e00\u4e2a\u6d88\u8d39\u7ec4\n        client_id =&gt; &quot;logstash1&quot;               # \u5ba2\u6237\u7aefID\n        fetch_max_wait_ms =&gt; &quot;1000&quot;            # \u6307\u5f53\u6ca1\u6709\u8db3\u591f\u7684\u6570\u636e\u7acb\u5373\u6ee1\u8db3fetch_min_bytes\u65f6\uff0c\u670d\u52a1\u5668\u5728\u56de\u7b54fetch\u8bf7\u6c42\u4e4b\u524d\u5c06\u963b\u585e\u7684\u6700\u957f\u65f6\u95f4\n  }\n}\n\n$ vim config\/02-output.conf\n\noutput {                                       # \u8f93\u51fa\u7ec4\u4ef6\n    elasticsearch {\n        # Logstash\u8f93\u51fa\u5230es\n        hosts =&gt; [&quot;172.20.166.25:9200&quot;, &quot;172.20.166.24:9200&quot;, &quot;172.20.166.22:9200&quot;, &quot;172.20.166.23:9200&quot;, &quot;172.20.166.26:9200&quot;]\n        index =&gt; &quot;%{[fields][source]}-%{+YYYY-MM-dd}&quot;      # \u76f4\u63a5\u5728\u65e5\u5fd7\u4e2d\u5339\u914d\uff0c\u7d22\u5f15\u4f1a\u53bb\u6389elk\n        # index =&gt; &quot;%{[@metadata][topic]}-%{+YYYY-MM-dd}&quot;  # \u4ee5\u65e5\u671f\u5efa\u7d22\u5f15\n        user =&gt; &quot;elastic&quot;\n        password =&gt; &quot;elastic123&quot;\n    }\n    #stdout {\n    #    codec =&gt; rubydebug\n    #}\n}\n\n$ vim config\/03-filter.conf\n\nfilter {\n   # \u5f53\u975e\u4e1a\u52a1\u5b57\u6bb5\u65f6\uff0c\u65e0traceId\u5219\u79fb\u9664\n   if ([message] =~ &quot;traceId=null&quot;) {          # \u8fc7\u6ee4\u7ec4\u4ef6\uff0c\u8fd9\u91cc\u53ea\u662f\u5c55\u793a\uff0c\u65e0\u5b9e\u9645\u610f\u4e49\uff0c\u6839\u636e\u81ea\u5df1\u7684\u4e1a\u52a1\u9700\u6c42\u8fdb\u884c\u8fc7\u6ee4\n      drop {}\n   }\n}\n\n# \u62f7\u8d1d\u914d\u7f6e\u5230 logstash2 logstash3 \u673a\u5668\u4e0a\n$ rsync -avp -e ssh \/data\/ELKStack\/logstash 172.20.166.28:\/data\/ELKStack\/\n$ rsync -avp -e ssh \/data\/ELKStack\/logstash 172.20.166.29:\/data\/ELKStack\/\n\n# \u542f\u52a8 logstash\n$ docker-compose up -d<\/code><\/pre>\n<h3>logstash2 \u64cd\u4f5c<\/h3>\n<pre><code class=\"language-bash\">$ cd \/data\/ELKStack\/logstash\n$ sed -i &#039;s\/logstash1\/logstash2\/g&#039; docker-compose.yml\n$ sed -i &#039;s\/logstash1\/logstash2\/g&#039; config\/01-input.conf\n\n# \u542f\u52a8 logstash\n$ docker-compose up -d<\/code><\/pre>\n<h3>logstash3 \u64cd\u4f5c<\/h3>\n<pre><code class=\"language-bash\">$ cd \/data\/ELKStack\/logstash\n$ sed -i &#039;s\/logstash1\/logstash3\/g&#039; docker-compose.yml\n$ sed -i &#039;s\/logstash1\/logstash3\/g&#039; config\/01-input.conf\n\n# \u542f\u52a8 logstash\n$ docker-compose up -d<\/code><\/pre>\n<h2>\u90e8\u7f72 filebeat<\/h2>\n<pre><code class=\"language-bash\"># \u914d\u7f6e filebeat yum\u6e90\uff0c\u8fd9\u91cc\u4ee5 centos7 \u4e3a\u4f8b\n$ rpm --import https:\/\/artifacts.elastic.co\/GPG-KEY-elasticsearch\n\n$ vim \/etc\/yum.repos.d\/elastic.repo\n\n[elastic-7.x]\nname=Elastic repository for 7.x packages\nbaseurl=https:\/\/artifacts.elastic.co\/packages\/7.x\/yum\ngpgcheck=1\ngpgkey=https:\/\/artifacts.elastic.co\/GPG-KEY-elasticsearch\nenabled=1\nautorefresh=1\ntype=rpm-md\n\n$ yum install -y filebeat-7.10.1\n$ systemctl enable filebeat\n\n# \u914d\u7f6e\n$ cd \/etc\/filebeat\/\n$ cp -a filebeat.yml filebeat.yml.old\n$ echo &gt; filebeat.yml\n\n# \u4ee5\u6536\u96c6nginx\u8bbf\u95ee\u65e5\u5fd7\u4e3a\u4f8b\n$ vim filebeat.yml\n\nfilebeat.inputs:                   # inputs\u4e3a\u590d\u6570\uff0c\u8868\u540dtype\u53ef\u4ee5\u6709\u591a\u4e2a\n- type: log                        # \u8f93\u5165\u7c7b\u578b\n  access:\n  enabled: true                    # \u542f\u7528\u8fd9\u4e2atype\u914d\u7f6e\n  json.keys_under_root: true       # \u9ed8\u8ba4\u8fd9\u4e2a\u503c\u662fFALSE\u7684\uff0c\u4e5f\u5c31\u662f\u6211\u4eec\u7684json\u65e5\u5fd7\u89e3\u6790\u540e\u4f1a\u88ab\u653e\u5728json\u952e\u4e0a\u3002\u8bbe\u4e3aTRUE\uff0c\u6240\u6709\u7684keys\u5c31\u4f1a\u88ab\u653e\u5230\u6839\u8282\u70b9\n  json.overwrite_keys: true        # \u662f\u5426\u8981\u8986\u76d6\u539f\u6709\u7684key\uff0c\u8fd9\u662f\u5173\u952e\u914d\u7f6e\uff0c\u5c06keys_under_root\u8bbe\u4e3aTRUE\u540e\uff0c\u518d\u5c06overwrite_keys\u4e5f\u8bbe\u4e3aTRUE\uff0c\u5c31\u80fd\u628afilebeat\u9ed8\u8ba4\u7684key\u503c\u7ed9\u8986\u76d6\n  max_bytes: 20480                 # \u5355\u6761\u65e5\u5fd7\u7684\u5927\u5c0f\u9650\u5236,\u5efa\u8bae\u9650\u5236(\u9ed8\u8ba4\u4e3a10M,queue.mem.events * max_bytes \u5c06\u662f\u5360\u6709\u5185\u5b58\u7684\u4e00\u90e8\u5206)\n  paths:\n    - \/var\/log\/nginx\/access.log    # \u76d1\u63a7nginx \u7684access\u65e5\u5fd7\n\n  fields:                          # \u989d\u5916\u7684\u5b57\u6bb5\n    source: nginx-access-prod      # \u81ea\u5b9a\u4e49source\u5b57\u6bb5\uff0c\u7528\u4e8ees\u5efa\u8bae\u7d22\u5f15\uff08\u5b57\u6bb5\u540d\u5c0f\u5199\uff0c\u6211\u8bb0\u5f97\u5927\u5199\u597d\u50cf\u4e0d\u884c\uff09\n\n# \u81ea\u5b9a\u4e49es\u7684\u7d22\u5f15\u9700\u8981\u628ailm\u8bbe\u7f6e\u4e3afalse\nsetup.ilm.enabled: false\n\noutput.kafka:            # \u8f93\u51fa\u5230kafka\n  enabled: true          # \u8be5output\u914d\u7f6e\u662f\u5426\u542f\u7528\n  hosts: [&quot;172.20.166.27:9092&quot;, &quot;172.20.166.28:9092&quot;, &quot;172.20.166.29:9092&quot;]  # kafka\u8282\u70b9\u5217\u8868\n  topic: &quot;elk-%{[fields.source]}&quot;   # kafka\u4f1a\u521b\u5efa\u8be5topic\uff0c\u7136\u540elogstash(\u53ef\u4ee5\u8fc7\u6ee4\u4fee\u6539)\u4f1a\u4f20\u7ed9es\u4f5c\u4e3a\u7d22\u5f15\u540d\u79f0\n  partition.hash:\n    reachable_only: true # \u662f\u5426\u53ea\u53d1\u5f80\u53ef\u8fbe\u5206\u533a\n  compression: gzip      # \u538b\u7f29\n  max_message_bytes: 1000000  # Event\u6700\u5927\u5b57\u8282\u6570\u3002\u9ed8\u8ba41000000\u3002\u5e94\u5c0f\u4e8e\u7b49\u4e8ekafka broker message.max.bytes\u503c\n  required_acks: 1  # kafka ack\u7b49\u7ea7\n  worker: 1  # kafka output\u7684\u6700\u5927\u5e76\u53d1\u6570\n  bulk_max_size: 2048    # \u5355\u6b21\u53d1\u5f80kafka\u7684\u6700\u5927\u4e8b\u4ef6\u6570\nlogging.to_files: true   # \u8f93\u51fa\u6240\u6709\u65e5\u5fd7\u5230file\uff0c\u9ed8\u8ba4true\uff0c \u8fbe\u5230\u65e5\u5fd7\u6587\u4ef6\u5927\u5c0f\u9650\u5236\u65f6\uff0c\u65e5\u5fd7\u6587\u4ef6\u4f1a\u81ea\u52a8\u9650\u5236\u66ff\u6362\uff0c\u8be6\u7ec6\u914d\u7f6e\uff1ahttps:\/\/www.cnblogs.com\/qinwengang\/p\/10982424.html\nclose_older: 30m         # \u5982\u679c\u4e00\u4e2a\u6587\u4ef6\u5728\u67d0\u4e2a\u65f6\u95f4\u6bb5\u5185\u6ca1\u6709\u53d1\u751f\u8fc7\u66f4\u65b0\uff0c\u5219\u5173\u95ed\u76d1\u63a7\u7684\u6587\u4ef6handle\u3002\u9ed8\u8ba41h\nforce_close_files: false # \u8fd9\u4e2a\u9009\u9879\u5173\u95ed\u4e00\u4e2a\u6587\u4ef6,\u5f53\u6587\u4ef6\u540d\u79f0\u7684\u53d8\u5316\u3002\u53ea\u5728window\u5efa\u8bae\u4e3atrue\n\n# \u6ca1\u6709\u65b0\u65e5\u5fd7\u91c7\u96c6\u540e\u591a\u957f\u65f6\u95f4\u5173\u95ed\u6587\u4ef6\u53e5\u67c4\uff0c\u9ed8\u8ba45\u5206\u949f\uff0c\u8bbe\u7f6e\u62101\u5206\u949f\uff0c\u52a0\u5feb\u6587\u4ef6\u53e5\u67c4\u5173\u95ed\nclose_inactive: 1m\n\n# \u4f20\u8f93\u4e863h\u540e\u834f\u6ca1\u6709\u4f20\u8f93\u5b8c\u6210\u7684\u8bdd\u5c31\u5f3a\u884c\u5173\u95ed\u6587\u4ef6\u53e5\u67c4\uff0c\u8fd9\u4e2a\u914d\u7f6e\u9879\u662f\u89e3\u51b3\u4ee5\u4e0a\u6848\u4f8b\u95ee\u9898\u7684key point\nclose_timeout: 3h\n\n# \u8fd9\u4e2a\u914d\u7f6e\u9879\u4e5f\u5e94\u8be5\u914d\u7f6e\u4e0a\uff0c\u9ed8\u8ba4\u503c\u662f0\u8868\u793a\u4e0d\u6e05\u7406\uff0c\u4e0d\u6e05\u7406\u7684\u610f\u601d\u662f\u91c7\u96c6\u8fc7\u7684\u6587\u4ef6\u63cf\u8ff0\u5728registry\u6587\u4ef6\u91cc\u6c38\u4e0d\u6e05\u7406\uff0c\u5728\u8fd0\u884c\u4e00\u6bb5\u65f6\u95f4\u540e\uff0cregistry\u4f1a\u53d8\u5927\uff0c\u53ef\u80fd\u4f1a\u5e26\u6765\u95ee\u9898\nclean_inactive: 72h\n\n# \u8bbe\u7f6e\u4e86clean_inactive\u540e\u5c31\u9700\u8981\u8bbe\u7f6eignore_older\uff0c\u4e14\u8981\u4fdd\u8bc1ignore_older &lt; clean_inactive\nignore_older: 70h\n\n# \u9650\u5236 CPU\u548c\u5185\u5b58\u8d44\u6e90\nmax_procs: 1 # \u9650\u5236\u4e00\u4e2aCPU\u6838\u5fc3,\u907f\u514d\u8fc7\u591a\u62a2\u5360\u4e1a\u52a1\u8d44\u6e90\nqueue.mem.events: 256 # \u5b58\u50a8\u4e8e\u5185\u5b58\u961f\u5217\u7684\u4e8b\u4ef6\u6570\uff0c\u6392\u961f\u53d1\u9001 (\u9ed8\u8ba44096)\nqueue.mem.flush.min_events: 128 # \u5c0f\u4e8e queue.mem.events ,\u589e\u52a0\u6b64\u503c\u53ef\u63d0\u9ad8\u541e\u5410\u91cf (\u9ed8\u8ba4\u503c2048)\n\n# \u542f\u52a8 filebeat\n$ systemctl start filebeat<\/code><\/pre>\n<h2>\u90e8\u7f72 curator\uff0c\u5b9a\u65f6\u6e05\u7406es\u7d22\u5f15<\/h2>\n<h3>logstash3 \u673a\u5668\u64cd\u4f5c<\/h3>\n<pre><code class=\"language-bash\"># \u53c2\u8003\u94fe\u63a5\uff1ahttps:\/\/www.elastic.co\/guide\/en\/elasticsearch\/client\/curator\/current\/yum-repository.html\n\n# \u5b89\u88c5 curator \u670d\u52a1\uff0c\u4ee5 centos7 \u4e3a\u4f8b\n$ rpm --import https:\/\/packages.elastic.co\/GPG-KEY-elasticsearch\n\n$ vim \/etc\/yum.repos.d\/elk-curator-5.repo\n\n[curator-5]\nname=CentOS\/RHEL 7 repository for Elasticsearch Curator 5.x packages\nbaseurl=https:\/\/packages.elastic.co\/curator\/5\/centos\/7\ngpgcheck=1\ngpgkey=https:\/\/packages.elastic.co\/GPG-KEY-elasticsearch\nenabled=1\n\n$ yum install elasticsearch-curator -y\n\n# \u521b\u5efa curator \u914d\u7f6e\u6587\u4ef6\u76ee\u5f55\u4e0e\u8f93\u51fa\u65e5\u5fd7\u76ee\u5f55\n$ mkdir -p \/data\/ELKStack\/curator\/logs\n$ cd \/data\/ELKStack\/curator\n\n$ vim config.yml\n\n---\n# Remember, leave a key empty if there is no value.  None will be a string,\n# # not a Python &quot;NoneType&quot;\nclient:\n  hosts: [&quot;172.20.166.25&quot;, &quot;172.20.166.24&quot;, &quot;172.20.166.22&quot;, &quot;172.20.166.23&quot;, &quot;172.20.166.26&quot;]\n  port: 9200\n  url_prefix:\n  use_ssl: False\n  certificate:\n  client_cert:\n  client_key:\n  ssl_no_validate: False\n  http_auth: elastic:elastic123\n  timeout: 150\n  master_only: False\n\nlogging:\n  loglevel: INFO\n  logfile: \/data\/ELKStack\/curator\/logs\/curator.log\n  logformat: default\n  blacklist: [&#039;elasticsearch&#039;, &#039;urllib3&#039;]\n\n$ vim action.yml\n\n---\n# Remember, leave a key empty if there is no value.  None will be a string,\n# not a Python &quot;NoneType&quot;\n#\n# Also remember that all examples have &#039;disable_action&#039; set to True.  If you\n# want to use this action as a template, be sure to set this to False after\n# copying it.\nactions:\n  1:\n    action: delete_indices\n    description: &gt;-\n      Delete indices older than 30 days. Ignore the error if the filter does not result in an actionable list of indices (ignore_empty_list) and exit cleanly.\n    options:\n      ignore_empty_list: True\n      disable_action: False\n    filters:\n    - filtertype: pattern\n      kind: regex\n      value: &#039;^((?!(kibana|json|monitoring|metadata|apm|async|transform|siem|security)).)*$&#039;\n    - filtertype: age\n      source: creation_date\n      direction: older\n      #timestring: &#039;%Yi-%m-%d&#039;\n      unit: days\n      unit_count: 30\n  2:\n    action: delete_indices\n    description: &gt;-\n      Delete indices older than 15 days. Ignore the error if the filter does not result in an actionable list of indices (ignore_empty_list) and exit cleanly.\n    options:\n      ignore_empty_list: True\n      disable_action: False\n    filters:\n    - filtertype: pattern\n      kind: regex\n      value: &#039;^(nginx-).*$&#039;\n    - filtertype: age\n      source: creation_date\n      direction: older\n      #timestring: &#039;%Yi-%m-%d&#039;\n      unit: days\n      unit_count: 15\n\n# \u8bbe\u7f6e\u5b9a\u65f6\u4efb\u52a1\u6e05\u7406es\u7d22\u5f15\n$ crontab -e\n\n0 0 * * * \/usr\/bin\/curator --config \/data\/ELKStack\/curator\/config.yml \/data\/ELKStack\/curator\/action.yml<\/code><\/pre>\n<h2>\u53c2\u8003\u94fe\u63a5<\/h2>\n<ul>\n<li>es\u8bc1\u4e66\u914d\u7f6e\uff1a<a target=\"_blank\" rel=\"noopener\" href=\"https:\/\/cloud.tencent.com\/developer\/article\/1549834\">https:\/\/cloud.tencent.com\/developer\/article\/1549834<\/a><\/li>\n<li>es\u5fd8\u8bb0\u5bc6\u7801\u627e\u56de\uff1a<a target=\"_blank\" rel=\"noopener\" href=\"https:\/\/www.cnblogs.com\/woshimrf\/p\/docker-es7.html\">https:\/\/www.cnblogs.com\/woshimrf\/p\/docker-es7.html<\/a><\/li>\n<li>es\u8bbe\u7f6e\u5bc6\u7801\uff1a<a target=\"_blank\" rel=\"noopener\" href=\"https:\/\/blog.csdn.net\/extraordinarylife\/article\/details\/107917764?utm_medium=distribute.pc_relevant.none-task-blog-baidujs_title-2&amp;spm=1001.2101.3001.4242\">https:\/\/blog.csdn.net\/extraordinarylife\/article\/details\/107917764?utm_medium=distribute.pc_relevant.none-task-blog-baidujs_title-2&#038;spm=1001.2101.3001.4242<\/a><\/li>\n<li>elk-kafka\u90e8\u7f72 1\uff1a<a target=\"_blank\" rel=\"noopener\" href=\"https:\/\/www.codenong.com\/cs106056710\/\">https:\/\/www.codenong.com\/cs106056710\/<\/a><\/li>\n<li>elk-kafka\u90e8\u7f72 2\uff1a<a target=\"_blank\" rel=\"noopener\" href=\"https:\/\/www.cnblogs.com\/lz0925\/p\/12061293.html\">https:\/\/www.cnblogs.com\/lz0925\/p\/12061293.html<\/a><\/li>\n<li>elk\u4f18\u5316\uff1a<a target=\"_blank\" rel=\"noopener\" href=\"https:\/\/www.clxz.top\/2020\/06\/19\/elk-kafka-optimization\/\">https:\/\/www.clxz.top\/2020\/06\/19\/elk-kafka-optimization\/<\/a><\/li>\n<li>es7\u7d22\u5f15\u5206\u7247\uff1a<a target=\"_blank\" rel=\"noopener\" href=\"https:\/\/www.elastic.co\/guide\/en\/elasticsearch\/reference\/current\/index-modules.html\">https:\/\/www.elastic.co\/guide\/en\/elasticsearch\/reference\/current\/index-modules.html<\/a><\/li>\n<li>filebeat\u4f18\u5316\uff1a<a target=\"_blank\" rel=\"noopener\" href=\"https:\/\/www.jianshu.com\/p\/389702465461\">https:\/\/www.jianshu.com\/p\/389702465461<\/a><\/li>\n<li>kafka\u6570\u636e\u4e0e\u65e5\u5fd7\u6e05\u7406\uff1a<a target=\"_blank\" rel=\"noopener\" href=\"https:\/\/blog.csdn.net\/VIP099\/article\/details\/106257561\">https:\/\/blog.csdn.net\/VIP099\/article\/details\/106257561<\/a><\/li>\n<\/ul>\n","protected":false},"excerpt":{"rendered":"<p>ELK\u67b6\u6784\u7b80\u4ecb \u9996\u5148logstash\u5177\u6709\u65e5\u5fd7\u91c7\u96c6\u3001\u8fc7\u6ee4\u3001\u7b5b\u9009\u7b49\u529f\u80fd\uff0c\u529f\u80fd\u5b8c\u5584\u4f46\u540c\u65f6\u4f53\u91cf\u4e5f\u4f1a\u6bd4\u8f83\u5927\uff0c\u6d88\u8017\u7cfb\u7edf\u8d44\u6e90 [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[179,345],"tags":[],"class_list":["post-1983","post","type-post","status-publish","format-standard","hentry","category-docker","category-elk"],"_links":{"self":[{"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/posts\/1983","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/comments?post=1983"}],"version-history":[{"count":0,"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/posts\/1983\/revisions"}],"wp:attachment":[{"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/media?parent=1983"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/categories?post=1983"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/tags?post=1983"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}