{"id":1377,"date":"2023-03-19T10:51:29","date_gmt":"2023-03-19T02:51:29","guid":{"rendered":"https:\/\/www.appblog.cn\/?p=1377"},"modified":"2023-04-28T21:13:29","modified_gmt":"2023-04-28T13:13:29","slug":"elk-deployment-records","status":"publish","type":"post","link":"https:\/\/www.appblog.cn\/index.php\/2023\/03\/19\/elk-deployment-records\/","title":{"rendered":"ELK\u90e8\u7f72\u8bb0\u5f55"},"content":{"rendered":"

Kafka<\/h2>\n

\u9996\u5148\u5b89\u88c5JDK\u73af\u5883\uff0c\u7136\u540e\u5b89\u88c5Kafka\u5e76\u521b\u5efatopic logstash<\/p>\n

\u5b98\u65b9\u4e0b\u8f7d\uff1ahttp:\/\/kafka.apache.org\/downloads<\/a><\/p>\n

<\/p>\n

# bin\/kafka-topics.sh --create --zookeeper 192.168.165.243:2181 --replication-factor 1 --partitions 1 --topic logstash<\/code><\/pre>\n
ElasticSearch<\/h2>\n
\u5b98\u65b9\u4e0b\u8f7d\uff1ahttps:\/\/www.elastic.co\/downloads\/elasticsearch<\/a> <\/p>\n
# tar -zxf elasticsearch-7.1.0-linux-x86_64.tar.gz -C \/data\/server\/\n# mv \/data\/server\/elasticsearch-7.1.0 \/data\/server\/elasticsearch\n# vim config\/elasticsearch.yml<\/code><\/pre>\n
network.host: 192.168.165.239  #\u8bbe\u7f6e\u8bbf\u95ee\u5730\u5740\u548c\u7aef\u53e3\u53f7\uff0c\u5426\u5219\u4e0d\u80fd\u5728\u6d4f\u89c8\u5668\u4e2d\u8bbf\u95ee\nhttp.port: 9200\n\n#cluster.name: es_cluster\nnode.name: node-1  #\u8bbe\u7f6eES\u96c6\u7fa4\u7684\u96c6\u7fa4\u540d\u79f0\uff0c\u4ee5\u53ca\u8fd9\u53f0\u673a\u5668\u5728\u96c6\u7fa4\u4e2d\u7684\u540d\u79f0\nnode.attr.rack: r1\n\npath.data: \/data\/server\/elasticsearch\/data  #\u8bbe\u7f6eES\u5b58\u50a8data\u548clog\u7684\u8def\u5f84\npath.logs: \/data\/logs\/elasticsearch\n\n#cluster.initial_master_nodes: ["node-1", "node-2"]\ncluster.initial_master_nodes: ["node-1"]<\/code><\/pre>\n
\n
\u6ce8\uff1aElasticsearch \u8981\u6c42\u4e0d\u80fd\u4f7f\u7528\u8d85\u7ea7\u7528\u6237root\u8fd0\u884c\uff0c\u6240\u4ee5\u6211\u4eec\u5efa\u7acb\u4e00\u4e2aes\u8d26\u53f7<\/p>\n<\/blockquote>\n
# \u521b\u5efaes\u8d26\u6237\nadduser es\n# \u4fee\u6539\u5bc6\u7801\npasswd es\n\n# \u4e3aesuser\u7528\u6237\u6388\u4e88elasticsearch\u76ee\u5f55\u6743\u9650\n# chown es -R \/data\/server\/elasticsearch<\/code><\/pre>\n
\u524d\u53f0\u542f\u52a8\uff1a<\/p>\n
# .\/bin\/elasticsearch<\/code><\/pre>\n
\u540e\u53f0\u542f\u52a8\uff1a<\/p>\n
# .\/bin\/elasticsearch -d<\/code><\/pre>\n
\u5728\u6d4f\u89c8\u5668\u4e2d\u8bbf\u95ee\uff1ahttp:\/\/192.168.16.20:9200\/<\/a><\/p>\n
{\n  "name" : "node-1",\n  "cluster_name" : "elasticsearch",\n  "cluster_uuid" : "_na_",\n  "version" : {\n    "number" : "7.1.0",\n    "build_flavor" : "default",\n    "build_type" : "tar",\n    "build_hash" : "606a173",\n    "build_date" : "2019-05-16T00:43:15.323135Z",\n    "build_snapshot" : false,\n    "lucene_version" : "8.0.0",\n    "minimum_wire_compatibility_version" : "6.8.0",\n    "minimum_index_compatibility_version" : "6.0.0-beta1"\n  },\n  "tagline" : "You Know, for Search"\n}<\/code><\/pre>\n
\u82e5\u62a5\u5982\u4e0b\u9519\u8bef\uff1a<\/p>\n
bound or publishing to a non-loopback address, enforcing bootstrap checks\nERROR: [3] bootstrap checks failed\n[1]: max file descriptors [4096] for elasticsearch process is too low, increase to at least [65535]\n[2]: max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]\n[3]: the default discovery settings are unsuitable for production use; at least one of [discovery.seed_hosts, discovery.seed_providers, cluster.initial_master_nodes] must be configured<\/code><\/pre>\n
\uff081\uff09max file descriptors [4096] for elasticsearch process is too low, increase to at least [65536]<\/p>\n
\u6bcf\u4e2a\u8fdb\u7a0b\u6700\u5927\u540c\u65f6\u6253\u5f00\u6587\u4ef6\u6570\u592a\u5c0f\uff0c\u53ef\u901a\u8fc7\u4e0b\u97622\u4e2a\u547d\u4ee4\u67e5\u770b\u5f53\u524d\u6570\u91cf<\/p>\n
ulimit -Hn\nulimit -Sn<\/code><\/pre>\n
\u4fee\u6539\/etc\/security\/limits.conf<\/code>\u6587\u4ef6\uff0c\u589e\u52a0\u914d\u7f6e\uff0c\u7528\u6237\u9000\u51fa\u540e\u91cd\u65b0\u767b\u5f55\u751f\u6548<\/p>\n
*               soft    nofile          65536\n*               hard    nofile          65536<\/code><\/pre>\n
\uff082\uff09max number of threads [3818] for user [es] is too low, increase to at least [4096]<\/p>\n
\u95ee\u9898\u540c\u4e0a\uff0c\u6700\u5927\u7ebf\u7a0b\u4e2a\u6570\u592a\u4f4e\u3002\u4fee\u6539\u914d\u7f6e\u6587\u4ef6\/etc\/security\/limits.conf<\/code>\uff0c\u589e\u52a0\u914d\u7f6e<\/p>\n
*               soft    nproc           4096\n*               hard    nproc           4096<\/code><\/pre>\n
\u53ef\u901a\u8fc7\u547d\u4ee4\u67e5\u770b<\/p>\n
ulimit -Hu\nulimit -Su<\/code><\/pre>\n
\uff083\uff09max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]<\/p>\n
\u4fee\u6539\/etc\/sysctl.conf<\/code>\u6587\u4ef6\uff0c\u5728\u672b\u5c3e\u589e\u52a0\u914d\u7f6evm.max_map_count=262144<\/code><\/p>\n
vi \/etc\/sysctl.conf\nsysctl -p<\/code><\/pre>\n
\u6267\u884c\u547d\u4ee4sysctl -p<\/code>\u751f\u6548<\/p>\n
Logstash<\/h2>\n
\u5b98\u65b9\u4e0b\u8f7d\uff1ahttps:\/\/www.elastic.co\/downloads\/logstash<\/a><\/p>\n
# tar -zxf logstash-7.1.0.tar.gz -C \/data\/server\/\n# mv \/data\/server\/logstash-7.1.0 \/data\/server\/logstash\n# cd \/data\/server\/logstash\/\n# mkdir config_file\n# vim config_file\/log.conf<\/code><\/pre>\n
\u524d\u53f0\u542f\u52a8\uff1a<\/p>\n
# bin\/logstash -f config_file\/log.conf<\/code><\/pre>\n
\u540e\u53f0\u542f\u52a8\uff1a<\/p>\n
# nohup bin\/logstash -f config_file\/log.conf >\/dev\/null &<\/code><\/pre>\n
\n
\u91c7\u96c6\u65e5\u5fd7\u6587\u4ef6\u5e76\u4f20\u5165Kafka\u7684log.conf\u914d\u7f6e<\/p>\n<\/blockquote>\n
input {\n    file {\n        path => ["\/home\/dubbo\/applogs\/*.log"]\n        type => "appblog"\n        start_position => beginning\n        #sincedb_path => "\/dev\/null"\n        #ignore_older => 0\n        codec => multiline {\n            pattern => "^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.\\d{3}"\n            negate => true\n            what => "previous"\n        }\n    }\n}\n\noutput {\n    kafka {\n        topic_id => "logstash"\n        bootstrap_servers => "192.168.16.20:9092"  # kafka\u7684\u5730\u5740\n        batch_size => 5\n        codec => json\n    }\n}<\/code><\/pre>\n
\n
\u63a5\u6536Kafka\u5e76\u4f20\u5165elasticsearch\u7684log.conf\u914d\u7f6e<\/p>\n<\/blockquote>\n
input{\n    kafka {\n        bootstrap_servers => "192.168.16.20:9092"\n        topics => "logstash"\n        group_id => "logstash"\n        consumer_threads => 5\n        decorate_events => true\n        codec => json\n        type => "appblog"\n        #auto_offset_reset => "smallest"\n        #reset_beginning => true\n   }\n}\n\nfilter {\n    if [type] == "appblog" {\n        if [message] =~ "^\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2}.\\d{3}\\s+\\[[a-zA-Z0-9._-]+\\]\\s+\\[[a-zA-Z0-9._-]+\\][\\s\\S]*$" {\n            grok {\n                patterns_dir => ".\/patterns"\n                add_field => {"logmatch" => "99999"}\n                match => { "message" => "%{TIME_STAMP_A:logtime}\\s+\\[%{APP_NAME:appname}\\]\\s+\\[%{LOG_LVL:loglvl}\\]\\s+\\[%{TRACE_ID:traceid}\\]\\s+\\[%{SPAN_ID:spanid}\\]" }\n            }\n            date {\n                match => ["logtime", "yyyy-MM-dd HH:mm:ss.SSS"]  \n                target => "messagetime"\n                #locale => "en"\n                #timezone => "+00:00"\n                #remove_field => ["logtime"]\n            }\n        }\n    }\n}\n\noutput {\n    elasticsearch {\n        hosts => ["192.168.16.20:9200"]\n        #hosts => ["192.168.16.20:9200","192.168.16.22:9200"]\n        index => "%{type}"\n    }\n}<\/code><\/pre>\n
Kibana<\/h2>\n
\u5b98\u65b9\u4e0b\u8f7d\uff1ahttps:\/\/www.elastic.co\/downloads\/kibana<\/a><\/p>\n
# tar -zxf kibana-7.1.0-linux-x86_64.tar.gz -C \/data\/server\/\n# mv \/data\/server\/kibana-7.1.0-linux-x86_64 \/data\/server\/kibana\n# cd \/data\/server\/kibana\/\n# vim config\/kibana.yml<\/code><\/pre>\n
server.port: 5601\nserver.host: "192.168.16.25"\nelasticsearch.hosts: ["http:\/\/192.168.16.20:9200"]\nxpack.reporting.encryptionKey: "yezhou"\nxpack.security.encryptionKey: "78C87E5FC3656BE577BB41A80F45F537"<\/code><\/pre>\n
\u524d\u53f0\u542f\u52a8\uff1a<\/p>\n
# .\/bin\/kibana<\/code><\/pre>\n
\u540e\u53f0\u542f\u52a8\uff1a<\/p>\n
# nohup .\/bin\/kibana >\/dev\/null &<\/code><\/pre>\n
\u5728\u6d4f\u89c8\u5668\u4e2d\u8bbf\u95ee\uff1a192.168.16.20:5601<\/code>\uff0c\u5373\u53ef\u8bbf\u95ee\u641c\u7d22<\/p>\n
\u67e5\u770bKibana\u8fdb\u7a0b\uff1a<\/p>\n
# ps -ef | grep node<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"
Kafka \u9996\u5148\u5b89\u88c5JDK\u73af\u5883\uff0c\u7136\u540e\u5b89\u88c5Kafka\u5e76\u521b\u5efatopic logstash \u5b98\u65b9\u4e0b\u8f7d\uff1ahttp:\/\/ […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[345],"tags":[180,342,183,344,343],"class_list":["post-1377","post","type-post","status-publish","format-standard","hentry","category-elk","tag-elasticsearch","tag-elk","tag-kafka","tag-kibana","tag-logstash"],"_links":{"self":[{"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/posts\/1377","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/comments?post=1377"}],"version-history":[{"count":0,"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/posts\/1377\/revisions"}],"wp:attachment":[{"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/media?parent=1377"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/categories?post=1377"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/tags?post=1377"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}