ELK Stack日志分析平台搭建实战：Elasticsearch索引管理与Kibana可视化查询

ELK Stack日志分析平台架构概述

网站运维中日志分析是故障排查和性能监控的核心环节。ELK Stack由Elasticsearch、Logstash、Kibana三个组件构成，加上Filebeat作为日志采集器，形成完整的日志收集、存储、分析和可视化链路。Elasticsearch作为分布式搜索引擎负责日志存储与全文检索，Logstash处理日志解析与字段提取，Kibana提供Web界面用于数据可视化与查询。整套架构支持每天TB级日志的实时检索，是DevOps实践中日志分析的事实标准方案。

Elasticsearch集群安装与配置

# 安装Elasticsearch 8.x（RPM方式）
rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch
cat > /etc/yum.repos.d/elasticsearch.repo << 'EOF'
[elasticsearch]
name=Elasticsearch repository for 8.x packages
baseURL=https://artifacts.elastic.co/packages/8.x/yum
gpgcheck=1
enabled=1
autorefresh=1
type=rpm-md
EOF

yum install -y elasticsearch

# 核心配置 elasticsearch.yml
cluster.name: elk-prod
node.name: node-1
network.host: 0.0.0.0
http.port: 9200
discovery.seed_hosts: ["10.0.1.101", "10.0.1.102", "10.0.1.103"]
cluster.initial_master_nodes: ["node-1", "node-2", "node-3"]

# JVM堆内存配置（不超过物理内存50%，不超过32GB）
# jvm.options
-Xms16g
-Xmx16g

# 启动并设置开机自启
systemctl enable elasticsearch
systemctl start elasticsearch

# 验证集群状态
curl -u elastic:password http://localhost:9200/_cluster/health?pretty
# "status": "green"
# "number_of_nodes": 3

Logstash日志解析管道配置

# 安装Logstash
yum install -y logstash

# 配置管道 logstash.conf
input {
  beats {
    port => 5044
    type => "nginx-access"
  }
}

filter {
  if [type] == "nginx-access" {
    grok {
      match => {
        "message" => '%{IPORHOST:client_ip} - %{DATA:user} \[%{HTTPDATE:timestamp}\] "%{WORD:method} %{URIPATHPARAM:uri} HTTP/%{NUMBER:http_version}" %{NUMBER:status_code} %{NUMBER:bytes} "%{DATA:referer}" "%{DATA:user_agent}" rt=%{NUMBER:request_time} uct=%{NUMBER:upstream_connect_time}'
      }
    }
    date {
      match => ["timestamp", "dd/MMM/yyyy:HH:mm:ss Z"]
      target => "@timestamp"
    }
    geoip {
      source => "client_ip"
      target => "geoip"
    }
    useragent {
      source => "user_agent"
      target => "ua"
    }
  }
}

output {
  elasticsearch {
    hosts => ["10.0.1.101:9200", "10.0.1.102:9200"]
    index => "nginx-access-%{+YYYY.MM.dd}"
    user => "elastic"
    password => "your_password"
  }
}

# 启动Logstash
systemctl start logstash

Filebeat轻量级日志采集部署

# 在每台应用服务器安装Filebeat
yum install -y filebeat

# 配置 filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  fields:
    log_type: nginx-access
  fields_under_root: true

- type: log
  enabled: true
  paths:
    - /var/log/nginx/error.log
  fields:
    log_type: nginx-error
  fields_under_root: true

# 多行合并（处理Java异常堆栈）
- type: log
  paths:
    - /var/log/app/application.log
  multiline.pattern: '^\d{4}-\d{2}-\d{2}'
  multiline.negate: true
  multiline.match: after

output.logstash:
  hosts: ["10.0.1.100:5044"]

# 启动Filebeat
systemctl enable filebeat
systemctl start filebeat

Elasticsearch索引模板与ILM生命周期管理

# 创建索引模板（自动为匹配的索引应用设置）
PUT _index_template/nginx_logs
{
  "index_patterns": ["nginx-access-*"],
  "template": {
    "settings": {
      "number_of_shards": 3,
      "number_of_replicas": 1,
      "index.lifecycle.name": "nginx-log-policy",
      "refresh_interval": "5s"
    },
    "mappings": {
      "properties": {
        "@timestamp": { "type": "date" },
        "client_ip": { "type": "ip" },
        "status_code": { "type": "integer" },
        "request_time": { "type": "float" },
        "uri": { "type": "keyword" },
        "geoip": {
          "properties": {
            "country_name": { "type": "keyword" },
            "city_name": { "type": "keyword" },
            "location": { "type": "geo_point" }
          }
        }
      }
    }
  }
}

# 配置ILM策略：日志保留30天
PUT _ilm/policy/nginx-log-policy
{
  "policy": {
    "phases": {
      "hot": {
        "actions": {
          "rollover": {
            "max_age": "1d",
            "max_size": "50gb"
          }
        }
      },
      "delete": {
        "min_age": "30d",
        "actions": {
          "delete": {}
        }
      }
    }
  }
}

Kibana可视化面板配置

# 安装Kibana
yum install -y kibana

# 配置 kibana.yml
server.port: 5601
server.host: "0.0.0.0"
elasticsearch.hosts: ["http://10.0.1.101:9200"]
elasticsearch.username: "kibana_system"
elasticsearch.password: "your_password"

# 启动Kibana
systemctl enable kibana
systemctl start kibana

# 访问 http://10.0.1.101:5601 创建Index Pattern
# Pattern: nginx-access-*  Time field: @timestamp

Kibana Dashboard构建与日志查询DSL语法

# Kibana Discover中常用KQL查询语法

# 按状态码筛选
status_code >= 500

# 组合查询
status_code: 404 and uri: "/api/*"

# 模糊匹配
message: "timeout" or message: "connection refused"

# 范围查询
request_time > 2.0

# 排除特定IP
not client_ip: "10.0.0.0/8"

# Elasticsearch DSL查询（API方式）
GET nginx-access-*/_search
{
  "query": {
    "bool": {
      "must": [
        { "range": { "status_code": { "gte": 500 } } }
      ],
      "filter": [
        { "range": { "@timestamp": { "gte": "now-1h" } } }
      ]
    }
  },
  "aggs": {
    "error_by_uri": {
      "terms": { "field": "uri", "size": 20 }
    }
  },
  "size": 0
}

# 响应时间百分位统计
GET nginx-access-*/_search
{
  "size": 0,
  "aggs": {
    "response_time_percentiles": {
      "percentiles": {
        "field": "request_time",
        "percents": [50, 95, 99, 99.9]
      }
    }
  }
}

监控告警集成与Kibana告警规则

Kibana内置Alerting功能可以基于Elasticsearch查询结果触发告警动作，无需额外安装Elasticsearch Watcher插件：

# 配置告警规则：5xx错误率超过阈值时告警
# Rule type: Elasticsearch Query
# 查询条件:
{
  "query": {
    "bool": {
      "must": [{ "range": { "status_code": { "gte": 500 } } }],
      "filter": [{ "range": { "@timestamp": { "gte": "now-5m" } } }]
    }
  }
}

# 告警条件: 文档数 > 100（5分钟内5xx超过100次）
# 告警动作: 发送Webhook到企业微信/钉钉/飞书
# Schedule: 每5分钟检查一次

# 响应时间告警：P95延迟超过2秒
{
  "size": 0,
  "aggs": {
    "p95_latency": {
      "percentiles": {
        "field": "request_time",
        "percents": [95]
      }
    }
  }
}
# 告警条件: p95_latency > 2.0

ELK平台性能调优与容量规划

生产环境中ELK性能瓶颈通常出现在Elasticsearch索引写入和磁盘I/O上。分片数量规划遵循单个分片不超过50GB原则，3节点集群的3分片1副本配置适合日均50GB以下的日志量。索引刷新间隔从默认1秒调整为5秒或10秒，可提升写入吞吐量30%以上。批量写入场景使用Bulk API替代单条写入，批量大小控制在5-15MB之间效果最佳。冷热数据分离架构中，使用ILM将7天前的索引迁移到冷数据节点（机械硬盘），热数据节点使用SSD保障实时查询性能。CI/CD流水线中可以将日志分析集成到部署后的自动化检查步骤，通过Kibana API拉取部署窗口内的错误日志，实现故障自动发现和回滚触发。