ELK Stack架构设计与组件选型
ELK Stack（Elasticsearch + Logstash + Kibana）是运维领域最广泛使用的日志分析方案。架构上Filebeat采集端部署于各业务节点，将日志推送到Logstash或直接写入Elasticsearch，Kibana负责可视化查询。中型集群建议加入Kafka作为消息缓冲层，防止日志高峰期Logstash处理能力不足导致数据丢失。
组件版本必须统一为8.x系列，Elasticsearch 8.x默认开启安全认证（TLS + 用户密码），相比7.x简化了安全配置流程。生产环境至少3个Elasticsearch节点组成集群，Logstash按吞吐需求水平扩展，Kibana单实例配合Nginx反向代理即可。
Elasticsearch集群部署与安全配置
Elasticsearch 8.x集群部署使用tar.gz或rpm方式，首次启动自动生成安全证书与默认密码。
# elasticsearch.yml 配置（节点1示例）
cluster.name: log-cluster
node.name: es-node-1
path.data: /data/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 0.0.0.0
http.port: 9200
transport.port: 9300
# 集群发现配置
discovery.seed_hosts: ["10.0.1.11", "10.0.1.12", "10.0.1.13"]
cluster.initial_master_nodes: ["es-node-1", "es-node-2", "es-node-3"]
# 安全配置
xpack.security.enabled: true
xpack.security.enrollment.enabled: true
xpack.security.http.ssl.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate# 创建索引生命周期策略（ILM）
PUT _ilm/policy/log-ilm-policy
{
"policy": {
"phases": {
"hot": {
"min_age": "0ms",
"actions": {
"rollover": { "max_age": "1d", "max_primary_shard_size": "50gb" },
"set_priority": { "priority": 100 }
}
},
"warm": {
"min_age": "7d",
"actions": {
"forcemerge": { "max_num_segments": 1 },
"shrink": { "number_of_shards": 1 },
"set_priority": { "priority": 50 }
}
},
"cold": {
"min_age": "30d",
"actions": {
"set_priority": { "priority": 0 },
"freeze": {}
}
},
"delete": {
"min_age": "90d",
"actions": { "delete": {} }
}
}
}
}Filebeat采集配置与Logstash管道处理
Filebeat作为轻量级日志采集器，资源占用极低，支持多行日志合并（如Java异常堆栈）和字段增强。
# filebeat.yml 配置
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/nginx/access.log
- /var/log/nginx/error.log
fields:
app: nginx
env: production
fields_under_root: true
multiline.pattern: '^\d{4}-\d{2}-\d{2}'
multiline.negate: true
multiline.match: after
output.kafka:
hosts: ["10.0.1.20:9092"]
topic: 'log-nginx'
partition.round_robin:
reachable_only: true
required_acks: 1# Logstash管道配置
input {
kafka {
bootstrap_servers => "10.0.1.20:9092"
topics => ["log-nginx"]
group_id => "logstash-nginx"
consumer_threads => 4
}
}
filter {
grok {
match => { "message" => '%{IP:client_ip} - - \[%{HTTPDATE:timestamp}\] "%{WORD:method} %{URIPATH:path} HTTP/%{NUMBER:http_version}" %{NUMBER:status} %{NUMBER:bytes}' }
}
date {
match => ["timestamp", "dd/MMM/yyyy:HH:mm:ss Z"]
target => "@timestamp"
}
mutate {
remove_field => ["message", "timestamp"]
convert => { "status" => "integer" "bytes" => "integer" }
}
}
output {
elasticsearch {
hosts => ["https://10.0.1.11:9200"]
index => "log-nginx-%{+YYYY.MM.dd}"
ilm_rollover_alias => "log-nginx"
ilm_policy => "log-ilm-policy"
user => "logstash_internal"
password => "${LOGSTASH_PASSWORD}"
}
}Kibana看板搭建与告警规则配置
Kibana提供Discover日志检索、Dashboard看板可视化和Alerting告警功能。针对Nginx日志场景，核心看板包括：请求量趋势图、状态码分布饼图、慢请求Top10、异常IP统计。
# Kibana告警规则
POST kbn:/api/alerting/rule
{
"rule_type_id": "es.query",
"name": "Nginx 5xx错误率告警",
"params": {
"index": ["log-nginx-*"],
"esqlQuery": "FROM log-nginx-* | WHERE status >= 500 | STATS error_count = COUNT(*) BY @timestamp | SORT @timestamp DESC",
"timeWindowSize": 300,
"timeWindowUnit": "s"
},
"schedule": { "interval": "1m" },
"actions": [{
"action_type_id": "webhook",
"params": { "url": "https://hooks.feishu.cn/xxx" }
}]
}生产环境ELK运维要点：索引模板配合ILM自动管理生命周期，避免手动维护索引；Elasticsearch JVM堆内存不超过31GB且为物理内存50%；冷热架构用SSD存储热数据、HDD存储冷数据；Kafka缓冲层保障日志高峰不丢数据；定期执行forcemerge减少段文件数提升查询性能。
原创文章，作者：小编，如若转载，请注明出处：https://www.yunthe.com/elkstack-ri-zhi-fen-xi-ping-tai-da-jian-yu-elasticsearch/
