{"id":1568,"date":"2023-03-25T17:42:59","date_gmt":"2023-03-25T09:42:59","guid":{"rendered":"https:\/\/www.appblog.cn\/?p=1568"},"modified":"2023-04-27T21:34:01","modified_gmt":"2023-04-27T13:34:01","slug":"filebeat-utilize-kafka-for-real-time-log-transmission","status":"publish","type":"post","link":"https:\/\/www.appblog.cn\/index.php\/2023\/03\/25\/filebeat-utilize-kafka-for-real-time-log-transmission\/","title":{"rendered":"Filebeat\u5229\u7528Kafka\u8fdb\u884c\u65e5\u5fd7\u5b9e\u65f6\u4f20\u8f93"},"content":{"rendered":"
vim filebeat.yml\nnohup .\/filebeat -c filebeat.yml &<\/code><\/pre>\n
<\/p>\n
#=========================== Filebeat inputs =============================\nfilebeat.inputs:\n- type: log\n  enabled: true\n  paths:\n    - \/var\/log\/applogs\/*.log\n  fields:\n    type: appblog\n  multiline:\n    pattern: ^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.\\d{3}\n    negate: true\n    match: after\n\n#================================ Outputs =====================================\noutput.kafka:\n  enabled: true\n  hosts: ["192.168.1.50:9092"]\n  topic: filebeat<\/code><\/pre>\n
192.168.1.50:9092<\/code>\u662f\u5355\u673akafka broker\uff0c\u5982\u679c\u662fkafka\u96c6\u7fa4\uff0c\u4f7f\u7528,<\/code>\u5206\u9694\u3002filebeat<\/code>\u662fkafka topic\uff0c\u9700\u6539\u6210\u5b9e\u9645\u60c5\u51b5\u7684\u503c\u3002\u53e6\u5916\u4ee5\u4e0b\u8fd9\u6bb5\u9700\u8981\u6ce8\u91ca\u6216\u5220\u9664\uff1a<\/p>\n
#-------------------------- Elasticsearch output ------------------------------\noutput.elasticsearch:\n  # Array of hosts to connect to.\n  hosts: ["localhost:9200"]\n\n  # Optional protocol and basic auth credentials.\n  #protocol: "https"\n  #username: "elastic"\n  #password: "changeme"<\/code><\/pre>\n
\u56e0\u4e3a\u5e76\u6ca1\u6709\u7528\u5230Elasticsearch\uff0c\u6240\u4ee5\u6709\u591a\u4e2a\u8f93\u51fa\u5728\u542f\u52a8filebeat\u65f6\u4f1a\u62a5\u9519\u3002<\/p>\n
\u6d88\u606f\u683c\u5f0f<\/p>\n
{\n    "@timestamp":"2019-11-29T04:36:59.633Z",\n    "@metadata":{\n        "beat":"filebeat",\n        "type":"_doc",\n        "version":"7.1.0",\n        "topic":"filebeat"\n    },\n    "input":{\n        "type":"log"\n    },\n    "host":{\n        "name":"ip-192-168-1-26.ap-southeast-1.compute.internal",\n        "hostname":"ip-192-168-1-26.ap-southeast-1.compute.internal",\n        "architecture":"x86_64",\n        "os":{\n            "platform":"amzn",\n            "version":"2",\n            "family":"redhat",\n            "name":"Amazon Linux",\n            "kernel":"4.14.114-105.126.amzn2.x86_64",\n            "codename":"Karoo"\n        },\n        "id":"2c6f141f3ddb47649a45fa62264a610e",\n        "containerized":true\n    },\n    "agent":{\n        "ephemeral_id":"276f7eba-c8fa-4b78-87b4-4d42d16f0a96",\n        "hostname":"ip-192-168-1-26.ap-southeast-1.compute.internal",\n        "id":"239a0097-87dd-4f40-9245-a98ec8bc671e",\n        "version":"7.1.0",\n        "type":"filebeat"\n    },\n    "ecs":{\n        "version":"1.0.0"\n    },\n    "cloud":{\n        "availability_zone":"ap-southeast-1c",\n        "instance":{\n            "id":"i-09f5e800dac65a867"\n        },\n        "machine":{\n            "type":"m5.large"\n        },\n        "region":"ap-southeast-1",\n        "provider":"aws"\n    },\n    "log":{\n        "offset":172973,\n        "file":{\n            "path":"\/var\/log\/applogs\/appblog-common.log"\n        }\n    },\n    "message":"The last packet sent successfully to the server was 0 milliseconds ago. The driver has not received any packets from the server."\n}<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"
vim filebeat.yml nohup .\/filebeat -c filebeat.yml & […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[345],"tags":[390,183],"class_list":["post-1568","post","type-post","status-publish","format-standard","hentry","category-elk","tag-filebeat","tag-kafka"],"_links":{"self":[{"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/posts\/1568","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/comments?post=1568"}],"version-history":[{"count":0,"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/posts\/1568\/revisions"}],"wp:attachment":[{"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/media?parent=1568"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/categories?post=1568"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.appblog.cn\/index.php\/wp-json\/wp\/v2\/tags?post=1568"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}