\u5b98\u7f51\uff1ahttps:\/\/www.elastic.co\/downloads\/beats\/filebeat<\/a><\/p>\n <\/p>\n \u53c2\u8003\uff1ahttps:\/\/github.com\/elastic\/beats\/issues\/11866<\/a> \u65e0\u6cd5\u81ea\u5b9a\u4e49\u7d22\u5f15\u540d\u79f0\u662f\u56e0\u4e3a\uff0c\u7d22\u5f15\u751f\u547d\u5468\u671f\u7ba1\u7406ilm\u529f\u80fd\u9ed8\u8ba4\u5f00\u542f\uff0c\u5f00\u542f\u7684\u60c5\u51b5\u4e0b\u7d22\u5f15\u540d\u79f0\u53ea\u80fd\u4e3a \u914d\u7f6e\u6a21\u677f\uff1ahttps:\/\/www.elastic.co\/guide\/en\/beats\/filebeat\/current\/ilm.html<\/a><\/p>\n <\/p>\n# wget https:\/\/artifacts.elastic.co\/downloads\/beats\/filebeat\/filebeat-7.1.0-linux-x86_64.tar.gz\n# tar -zxf filebeat-7.1.0-linux-x86_64.tar.gz -C \/usr\/local\/\n# mv \/usr\/local\/filebeat-7.1.0-linux-x86_64 \/usr\/local\/filebeat\n# cd \/usr\/local\/filebeat\/ \n# vim filebeat.yml<\/code><\/pre>\nFilebeat\u914d\u7f6e<\/h2>\n
filebeat.yml<\/h3>\n
\n\u53c2\u8003\uff1ahttps:\/\/iminto.github.io\/post\/filebeat\u4fee\u6539index\u7684\u4e00\u4e2a\u5751\/<\/a><\/p>\n#=========================== Filebeat inputs =============================\nfilebeat.inputs:\n- type: log\n enabled: false\n paths:\n - \/usr\/local\/nginx\/logs\/access.log\n #scan_frequency: 10s\n\n#============================= Filebeat modules ===============================\nfilebeat.config.modules:\n path: ${path.config}\/modules.d\/*.yml\n reload.enabled: false\n\n#==================== Elasticsearch template setting ==========================\n# 7.x\u7684\u7248\u672c\u4e2d\u9700\u8981\u7981\u7528\u6b64\u7d22\u5f15\u751f\u547d\u5468\u671f\uff0c\u5426\u5219\u5728\u6307\u5b9aes\u7d22\u5f15\u540d\u5b57\u7684\u65f6\u5019\u4f1a\u6709\u95ee\u9898\nsetup.ilm.enabled: false\n# \u6dfb\u52a0\u6a21\u677f\u914d\u7f6e\uff0c\u5426\u5219\u65e0\u6cd5\u6307\u5b9aes\u7684\u7d22\u5f15\u540d\nsetup.template.enabled: true\nsetup.template.name: "nginx-log"\nsetup.template.pattern: "nginx-log-*"\nsetup.template.overwrite: true\n\nsetup.template.settings:\n index.number_of_shards: 1\n index.number_of_replicas: 1\n\n#============================== Kibana =====================================\nsetup.kibana:\n host: "192.168.165.239:5601"\n\n#================================ Outputs =====================================\noutput.elasticsearch:\n hosts: ["192.168.16.20:9200"] # ["192.168.16.21:9200", "192.168.16.22:9200", "192.168.16.23:9200"]\n index: "nginx-log-%{+yyyy.MM.dd}"\n\n#================================ Processors =====================================\nprocessors:\n #- add_host_metadata: ~\n #- add_cloud_metadata: ~\n - drop_fields:\n fields: ["beat.name", "beat.version", "host.architecture","host.architecture","host.name","beat.hostname","log.file.path"]<\/code><\/pre>\nfilebeat-*<\/code>\uff0c \u901a\u8fc7setup.ilm.enabled: false<\/code>\u8fdb\u884c\u5173\u95ed\uff1b\u5982\u679c\u8981\u4f7f\u7528\u81ea\u5b9a\u4e49\u7684\u7d22\u5f15\u540d\u79f0\uff0c\u540c\u65f6\u53c8\u9700\u8981\u542f\u7528ilm\uff0c\u53ef\u4ee5\u4fee\u6539filebeat\u7684\u6a21\u677f<\/p>\nsetup.ilm.enabled: auto\nsetup.ilm.rollover_alias: "filebeat"\nsetup.ilm.pattern: "{now\/d}-000001"<\/code><\/pre>\n\u542f\u7528\u6a21\u5757nginx<\/h3>\n
# cp modules.d\/nginx.yml.disabled modules.d\/nginx.yml\n# vim modules.d\/nginx.yml<\/code><\/pre>\n# Module: nginx\n# Docs: https:\/\/www.elastic.co\/guide\/en\/beats\/filebeat\/7.1\/filebeat-module-nginx.html\n\n- module: nginx\n # Access logs\n access:\n enabled: true\n\n # Set custom paths for the log files. If left empty,\n # Filebeat will choose the paths depending on your OS.\n var.paths: ["\/usr\/local\/nginx\/logs\/access.log"]\n\n # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.\n #var.convert_timezone: true\n\n # Error logs\n error:\n enabled: true\n\n # Set custom paths for the log files. If left empty,\n # Filebeat will choose the paths depending on your OS.\n var.paths: ["\/usr\/local\/nginx\/logs\/error.log"]\n\n # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.\n #var.convert_timezone: true<\/code><\/pre>\nNginx\u914d\u7f6e<\/h3>\n
http {\n log_format main '$remote_addr -'\n ' $remote_user'\n ' [$time_local]'\n ' "$request"'\n ' $status'\n ' $body_bytes_sent'\n ' "$http_referer"'\n ' "$http_user_agent"'\n ' "$http_x_forwarded_for"'\n ' $upstream_response_time'\n ' $upstream_addr';\n\n access_log logs\/access.log main;\n}<\/code><\/pre>\ningest\u914d\u7f6e<\/h3>\n
# cp module\/nginx\/access\/ingest\/default.json module\/nginx\/access\/ingest\/default.json.bak\n# vim module\/nginx\/access\/ingest\/default.json<\/code><\/pre>\n{\n "description": "Pipeline for parsing Nginx access logs. Requires the geoip and user_agent plugins.",\n "processors": [{\n "grok": {\n "field": "message",\n "patterns":[\n "\\"?%{IP_LIST:nginx.access.remote_ip_list} - %{DATA:nginx.access.user_name} \\\\[%{HTTPDATE:nginx.access.time}\\\\] \\"%{GREEDYDATA:nginx.access.info}\\" %{NUMBER:nginx.access.response_code} %{NUMBER:nginx.access.body_sent.bytes} \\"%{DATA:nginx.access.referrer}\\" \\"%{DATA:nginx.access.agent}\\" \\"%{GREEDYDATA:nginx.access.xforwardedfor}\\" %{GREEDYDATA:nginx.access.upstream_response_time} %{GREEDYDATA:nginx.access.upstream_addr}"\n ],\n "pattern_definitions": {\n "IP_LIST": "%{IP}(\\"?,?\\\\s*%{IP})*"\n },\n "ignore_missing": true\n }\n }, {\n "grok": {\n "field": "nginx.access.info",\n "patterns": [\n "%{WORD:nginx.access.method} %{DATA:nginx.access.url} HTTP\/%{NUMBER:nginx.access.http_version}",\n ""\n ],\n "ignore_missing": true\n }\n }, {\n "remove": {\n "field": "nginx.access.info"\n }\n }, {\n "split": {\n "field": "nginx.access.remote_ip_list",\n "separator": "\\"?,?\\\\s+"\n }\n }, {\n "script": {\n "lang": "painless",\n "inline": "boolean isPrivate(def ip) { try { StringTokenizer tok = new StringTokenizer(ip, '.'); int firstByte = Integer.parseInt(tok.nextToken()); int secondByte = Integer.parseInt(tok.nextToken()); if (firstByte == 10) { return true; } if (firstByte == 192 && secondByte == 168) { return true; } if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) { return true; } if (firstByte == 127) { return true; } return false; } catch (Exception e) { return false; } } def found = false; for (def item : ctx.nginx.access.remote_ip_list) { if (!isPrivate(item)) { ctx.nginx.access.remote_ip = item; found = true; break; } } if (!found) { ctx.nginx.access.remote_ip = ctx.nginx.access.remote_ip_list[0]; }"\n }\n }, {\n "remove":{\n "field": "message"\n }\n }, {\n "rename": {\n "field": "@timestamp",\n "target_field": "read_timestamp"\n }\n }, {\n "date": {\n "field": "nginx.access.time",\n "target_field": "@timestamp",\n "formats": ["dd\/MMM\/YYYY:H:m:s Z"]\n }\n },{\n "remove": {\n "field": "nginx.access.time"\n }\n }, {\n "user_agent": {\n "field": "nginx.access.agent",\n "target_field": "nginx.access.user_agent"\n }\n }, {\n "rename": {\n "field": "nginx.access.agent",\n "target_field": "nginx.access.user_agent.original"\n }\n }, {\n "geoip": {\n "field": "nginx.access.remote_ip",\n "target_field": "nginx.access.geoip"\n }\n }, {\n "script": {\n "lang": "painless",\n "inline": "String tmp=ctx.nginx.access.upstream_response_time; if (tmp=='-'){ctx.nginx.access.upstream_response_time=-1.0}else{ctx.nginx.access.upstream_response_time=Float.parseFloat(tmp)}"\n }\n }],\n "on_failure" : [{\n "set" : {\n "field" : "error.message",\n "value" : "{{ _ingest.on_failure_message }}"\n }\n }]\n}<\/code><\/pre>\nFilebeat\u542f\u52a8<\/h2>\n
nohup .\/filebeat -e -c filebeat.yml >&\/dev\/null &<\/code><\/pre>\nGrafana\u914d\u7f6e<\/h2>\n
Data Source<\/h3>\n
Dashboard<\/h3>\n