ElasticSearch 排障常用方法

1，集群状态，节点在线情况，集群参数配置

GET _cluster/health?pretty
# {
   
#   "cluster_name" : "my-es",
#   "status" : "green",
#   "timed_out" : false,
#   "number_of_nodes" : 3,
#   "number_of_data_nodes" : 3,
#   "active_primary_shards" : 917,
#   "active_shards" : 1834,
#   "relocating_shards" : 0,
#   "initializing_shards" : 0,
#   "unassigned_shards" : 0,
#   "delayed_unassigned_shards" : 0,
#   "number_of_pending_tasks" : 0,
#   "number_of_in_flight_fetch" : 0,
#   "task_max_waiting_in_queue_millis" : 0,
#   "active_shards_percent_as_number" : 100.0
# }

# 查看帮助：GET _cat/nodes?help
GET _cat/nodes?v
GET _cat/nodes?v&h=ip,name,version,disk.used_percent,ram.percent,cpu,file_desc.current,file_desc.max   
# ip           name                version disk.used_percent ram.percent cpu file_desc.current file_desc.max
# 192.56.0.108 master-192.56.0.108 7.5.1               38.64          98   8             10894         65535
# 192.56.0.107 master-192.56.0.107 7.5.1               38.64          98   6             10895         65535
# 192.56.0.106 master-192.56.0.106 7.5.1               38.87          99   5             10998         65535
GET _nodes/stats/process?filter_path=**.max_file_descriptors
# {
   
#   "nodes" : {
   
#     "nXQsMpumSsC5zT818W7c4A" : {
   
#       "process" : {
   
#         "max_file_descriptors" : 65535
#       }
#     },
# ...
# }



#查看集群配置参数
GET _cluster/settings?include_defaults=true&pretty
GET _cluster/settings?pretty
# {
   
#   "persistent" : {
   
#     "cluster" : {
   
#       "max_shards_per_node" : "10000"
#     },
#     "search" : {
   
#       "max_buckets" : "100000"
#     }
#   },
#   "transient" : { }
# }

2，查看异常索引、分片，分析异常原因，手动分配分片

• 可能的问题： 查看集群状态，提示unassigned_shards 很多
• 在kibana 可查看到某些索引状态为red

# 查看帮助：GET _cat/indices?help
#GET _cat/indices?v
# 查看所有异常索引
GET _cat/indices?health=red

GET _cat/shards?v
GET _cat/shards?v&h=index,shard,state,unassigned.reason
GET _cat/shards/linux_2023_11?v
# index             shard prirep state        docs   store ip           node
# linux_2023_11 5     p      STARTED 867480195 228.1gb 192.56.0.107 master-192.56.0.107
# linux_2023_11 5     r      STARTED 867480195 228.1gb 192.56.0.108 master-192.56.0.108

#找到分片状态为UNASSIGNED( 非STARTED ), 对未分配的分片序号，查看原因
GET _cluster/allocation/explain
{
   
  "index":"linux_2023_11",
  "shard":0,
  "primary":true
}

#统一设置，重新分配分片
POST _cluster/reroute?retry_failed=true
#如果还是该索引还是red, 分配不了,可以尝试手动重新分配
POST  /_cluster/reroute
{
   
  "commands": [
    {
   
      "allocate_empty_primary": {
   
        "index": "linux_2023_11",
        "shard": 1,
        "node": "master-192.56.0.107",
        "accept_data_loss": true
      }
    }
  ]
}