Versions (relevant - OpenSearch/Dashboard/Server OS/Browser):
curl -u $OS_UN:$OS_PW -s "https://localhost:9200/"
{
"name" : "opensearch-001",
"cluster_name" : "opensearch-cluster",
"cluster_uuid" : "0Do19PnlSfyoMA9e5PnS9Q",
"version" : {
"distribution" : "opensearch",
"number" : "2.19.3",
"build_type" : "tar",
"build_hash" : "a90f864b8524bc75570a8461ccb569d2a4bfed42",
"build_date" : "2025-07-21T22:34:18.003652598Z",
"build_snapshot" : false,
"lucene_version" : "9.12.2",
"minimum_wire_compatibility_version" : "7.10.0",
"minimum_index_compatibility_version" : "7.0.0"
},
"tagline" : "The OpenSearch Project: https://opensearch.org/"
}
Describe the issue:
After a power outage last week I got back to work to find my OpenSearch-cluster in YELLOW-state.
I noticed that some indices were “stuck” so I raised the limit of recoveries, that sometimes gets things moving.
PUT _cluster/settings
{
"transient" : {
"cluster.routing.allocation.node_concurrent_incoming_recoveries" : 10,
"cluster.routing.allocation.node_concurrent_outgoing_recoveries" : 10
}
}
But i still have a couple of shards that are stuck.
What do I do now?
Configuration:
GET _cluster/settings
{
"persistent": {
"cluster": {
"routing": {
"allocation": {
"enable": "all"
}
}
},
"plugins": {
"index_state_management": {
"metadata_migration": {
"status": "1"
},
"template_migration": {
"control": "-1"
}
}
},
"logger": {
"org": {
"elasticsearch": {
"indices": {
"recovery": "TRACE"
}
},
"opensearch": {
"indexmanagement": {
"indexstatemanagement": {
"ManagedIndexRunner": "WARN"
}
},
"jobscheduler": {
"scheduler": {
"JobScheduler": "WARN"
}
}
}
}
}
},
"transient": {
"cluster": {
"routing": {
"allocation": {
"node_concurrent_incoming_recoveries": "10",
"node_concurrent_outgoing_recoveries": "10"
}
}
}
}
}
Relevant Logs or Screenshots:
curl -u $OS_UN:$OS_PW -s "https://localhost:9200/_cluster/health" | jq
{
"cluster_name": "opensearch-cluster",
"status": "yellow",
"timed_out": false,
"number_of_nodes": 3,
"number_of_data_nodes": 3,
"discovered_master": true,
"discovered_cluster_manager": true,
"active_primary_shards": 291,
"active_shards": 576,
"relocating_shards": 0,
"initializing_shards": 19,
"unassigned_shards": 0,
"delayed_unassigned_shards": 0,
"number_of_pending_tasks": 0,
"number_of_in_flight_fetch": 0,
"task_max_waiting_in_queue_millis": 0,
"active_shards_percent_as_number": 96.80672268907563
}
curl -u $OS_UN:$OS_PW -s "https://localhost:9200/_cat/shards?h=index,shard,prirep,state,unassigned.reason" | grep STARTED -v
.opendistro-ism-managed-index-history-2025.12.30-000727 0 r INITIALIZING NODE_LEFT
.opendistro-ism-managed-index-history-2025.12.25-000722 0 r INITIALIZING NODE_LEFT
.ds-logstash-MASKED-000134 0 r INITIALIZING NODE_LEFT
.opendistro-ism-managed-index-history-2025.12.19-000716 0 r INITIALIZING NODE_LEFT
security-auditlog-2025.09.03 0 r INITIALIZING NODE_LEFT
.ds-logstash-MASKED-000015 0 r INITIALIZING NODE_LEFT
security-auditlog-2025.09.15 0 r INITIALIZING NODE_LEFT
.ds-logstash-MASKED-000001 0 r INITIALIZING NODE_LEFT
.opendistro-ism-managed-index-history-2025.12.16-000713 0 r INITIALIZING NODE_LEFT
security-auditlog-2025.11.26 0 r INITIALIZING NODE_LEFT
.ds-logstash-MASKED-000001 0 r INITIALIZING NODE_LEFT
security-auditlog-2025.12.08 0 r INITIALIZING NODE_LEFT
security-auditlog-2025.12.02 0 r INITIALIZING NODE_LEFT
.ds-logstash-MASKED-000011 0 r INITIALIZING NODE_LEFT
.opendistro-ism-managed-index-history-2025.12.22-000719 0 r INITIALIZING NODE_LEFT
.ds-logstash-MASKED-000001 0 r INITIALIZING NODE_LEFT
security-auditlog-2025.12.14 0 r INITIALIZING NODE_LEFT
.opendistro-ism-managed-index-history-2025.12.28-000725 0 r INITIALIZING NODE_LEFT
security-auditlog-2025.12.20 0 r INITIALIZING NODE_LEFT
curl -u $OS_UN:$OS_PW -s "https://localhost:9200/_recovery?active_only=true" | jq
{
".ds-logstash-MASKED-000134": {
"shards": [
{
"id": 0,
"type": "PEER",
"stage": "INIT",
"primary": false,
"start_time_in_millis": 1768231641739,
"total_time_in_millis": 1600942,
"source": {
"id": "-8b2ZEInT16hb9U6KzkfRg",
"host": "opensearch-002",
"transport_address": "10.10.10.2:9300",
"ip": "10.10.10.2",
"name": "opensearch-002"
},
"target": {
"id": "B4XkLHTXRmW7ePYuccoN9g",
"host": "opensearch-003",
"transport_address": "10.10.10.3:9300",
"ip": "10.10.10.3",
"name": "opensearch-003"
},
"index": {
"size": {
"total_in_bytes": 0,
"reused_in_bytes": 0,
"recovered_in_bytes": 0,
"percent": "0.0%"
},
"files": {
"total": 0,
"reused": 0,
"recovered": 0,
"percent": "0.0%"
},
"total_time_in_millis": 0,
"source_throttle_time_in_millis": 0,
"target_throttle_time_in_millis": 0
},
"translog": {
"recovered": 0,
"total": -1,
"percent": "-1.0%",
"total_on_start": -1,
"total_time_in_millis": 0
},
"verify_index": {
"check_index_time_in_millis": 0,
"total_time_in_millis": 0
}
}
]
},
"security-auditlog-2025.09.03": {
"shards": [
{
"id": 0,
"type": "PEER",
"stage": "INIT",
"primary": false,
"start_time_in_millis": 1768231649814,
"total_time_in_millis": 1592867,
"source": {
"id": "-8b2ZEInT16hb9U6KzkfRg",
"host": "opensearch-002",
"transport_address": "10.10.10.2:9300",
"ip": "10.10.10.2",
"name": "opensearch-002"
},
"target": {
"id": "B4XkLHTXRmW7ePYuccoN9g",
"host": "opensearch-003",
"transport_address": "10.10.10.3:9300",
"ip": "10.10.10.3",
"name": "opensearch-003"
},
"index": {
"size": {
"total_in_bytes": 0,
"reused_in_bytes": 0,
"recovered_in_bytes": 0,
"percent": "0.0%"
},
"files": {
"total": 0,
"reused": 0,
"recovered": 0,
"percent": "0.0%"
},
"total_time_in_millis": 0,
"source_throttle_time_in_millis": 0,
"target_throttle_time_in_millis": 0
},
"translog": {
"recovered": 0,
"total": -1,
"percent": "-1.0%",
"total_on_start": -1,
"total_time_in_millis": 0
},
"verify_index": {
"check_index_time_in_millis": 0,
"total_time_in_millis": 0
}
}
]
},
".ds-logstash-MASKED-000015": {
"shards": [
{
"id": 0,
"type": "PEER",
"stage": "INIT",
"primary": false,
"start_time_in_millis": 1768231646034,
"total_time_in_millis": 1596647,
"source": {
"id": "-8b2ZEInT16hb9U6KzkfRg",
"host": "opensearch-002",
"transport_address": "10.10.10.2:9300",
"ip": "10.10.10.2",
"name": "opensearch-002"
},
"target": {
"id": "B4XkLHTXRmW7ePYuccoN9g",
"host": "opensearch-003",
"transport_address": "10.10.10.3:9300",
"ip": "10.10.10.3",
"name": "opensearch-003"
},
"index": {
"size": {
"total_in_bytes": 0,
"reused_in_bytes": 0,
"recovered_in_bytes": 0,
"percent": "0.0%"
},
"files": {
"total": 0,
"reused": 0,
"recovered": 0,
"percent": "0.0%"
},
"total_time_in_millis": 0,
"source_throttle_time_in_millis": 0,
"target_throttle_time_in_millis": 0
},
"translog": {
"recovered": 0,
"total": -1,
"percent": "-1.0%",
"total_on_start": -1,
"total_time_in_millis": 0
},
"verify_index": {
"check_index_time_in_millis": 0,
"total_time_in_millis": 0
}
}
]
},
"security-auditlog-2025.09.15": {
"shards": [
{
"id": 0,
"type": "PEER",
"stage": "INIT",
"primary": false,
"start_time_in_millis": 1768231650108,
"total_time_in_millis": 1592573,
"source": {
"id": "-8b2ZEInT16hb9U6KzkfRg",
"host": "opensearch-002",
"transport_address": "10.10.10.2:9300",
"ip": "10.10.10.2",
"name": "opensearch-002"
},
"target": {
"id": "B4XkLHTXRmW7ePYuccoN9g",
"host": "opensearch-003",
"transport_address": "10.10.10.3:9300",
"ip": "10.10.10.3",
"name": "opensearch-003"
},
"index": {
"size": {
"total_in_bytes": 0,
"reused_in_bytes": 0,
"recovered_in_bytes": 0,
"percent": "0.0%"
},
"files": {
"total": 0,
"reused": 0,
"recovered": 0,
"percent": "0.0%"
},
"total_time_in_millis": 0,
"source_throttle_time_in_millis": 0,
"target_throttle_time_in_millis": 0
},
"translog": {
"recovered": 0,
"total": -1,
"percent": "-1.0%",
"total_on_start": -1,
"total_time_in_millis": 0
},
"verify_index": {
"check_index_time_in_millis": 0,
"total_time_in_millis": 0
}
}
]
},
".ds-logstash-MASKED-000001": {
"shards": [
{
"id": 0,
"type": "PEER",
"stage": "INIT",
"primary": false,
"start_time_in_millis": 1768231651680,
"total_time_in_millis": 1591001,
"source": {
"id": "-8b2ZEInT16hb9U6KzkfRg",
"host": "opensearch-002",
"transport_address": "10.10.10.2:9300",
"ip": "10.10.10.2",
"name": "opensearch-002"
},
"target": {
"id": "B4XkLHTXRmW7ePYuccoN9g",
"host": "opensearch-003",
"transport_address": "10.10.10.3:9300",
"ip": "10.10.10.3",
"name": "opensearch-003"
},
"index": {
"size": {
"total_in_bytes": 0,
"reused_in_bytes": 0,
"recovered_in_bytes": 0,
"percent": "0.0%"
},
"files": {
"total": 0,
"reused": 0,
"recovered": 0,
"percent": "0.0%"
},
"total_time_in_millis": 0,
"source_throttle_time_in_millis": 0,
"target_throttle_time_in_millis": 0
},
"translog": {
"recovered": 0,
"total": -1,
"percent": "-1.0%",
"total_on_start": -1,
"total_time_in_millis": 0
},
"verify_index": {
"check_index_time_in_millis": 0,
"total_time_in_millis": 0
}
}
]
},
"security-auditlog-2025.11.26": {
"shards": [
{
"id": 0,
"type": "PEER",
"stage": "INIT",
"primary": false,
"start_time_in_millis": 1768231641853,
"total_time_in_millis": 1600828,
"source": {
"id": "-8b2ZEInT16hb9U6KzkfRg",
"host": "opensearch-002",
"transport_address": "10.10.10.2:9300",
"ip": "10.10.10.2",
"name": "opensearch-002"
},
"target": {
"id": "B4XkLHTXRmW7ePYuccoN9g",
"host": "opensearch-003",
"transport_address": "10.10.10.3:9300",
"ip": "10.10.10.3",
"name": "opensearch-003"
},
"index": {
"size": {
"total_in_bytes": 0,
"reused_in_bytes": 0,
"recovered_in_bytes": 0,
"percent": "0.0%"
},
"files": {
"total": 0,
"reused": 0,
"recovered": 0,
"percent": "0.0%"
},
"total_time_in_millis": 0,
"source_throttle_time_in_millis": 0,
"target_throttle_time_in_millis": 0
},
"translog": {
"recovered": 0,
"total": -1,
"percent": "-1.0%",
"total_on_start": -1,
"total_time_in_millis": 0
},
"verify_index": {
"check_index_time_in_millis": 0,
"total_time_in_millis": 0
}
}
]
},
".ds-logstash-MASKED-000001": {
"shards": [
{
"id": 0,
"type": "PEER",
"stage": "INIT",
"primary": false,
"start_time_in_millis": 1768231651778,
"total_time_in_millis": 1590903,
"source": {
"id": "-8b2ZEInT16hb9U6KzkfRg",
"host": "opensearch-002",
"transport_address": "10.10.10.2:9300",
"ip": "10.10.10.2",
"name": "opensearch-002"
},
"target": {
"id": "B4XkLHTXRmW7ePYuccoN9g",
"host": "opensearch-003",
"transport_address": "10.10.10.3:9300",
"ip": "10.10.10.3",
"name": "opensearch-003"
},
"index": {
"size": {
"total_in_bytes": 0,
"reused_in_bytes": 0,
"recovered_in_bytes": 0,
"percent": "0.0%"
},
"files": {
"total": 0,
"reused": 0,
"recovered": 0,
"percent": "0.0%"
},
"total_time_in_millis": 0,
"source_throttle_time_in_millis": 0,
"target_throttle_time_in_millis": 0
},
"translog": {
"recovered": 0,
"total": -1,
"percent": "-1.0%",
"total_on_start": -1,
"total_time_in_millis": 0
},
"verify_index": {
"check_index_time_in_millis": 0,
"total_time_in_millis": 0
}
}
]
},
"security-auditlog-2025.12.08": {
"shards": [
{
"id": 0,
"type": "PEER",
"stage": "INIT",
"primary": false,
"start_time_in_millis": 1768231641932,
"total_time_in_millis": 1600749,
"source": {
"id": "-8b2ZEInT16hb9U6KzkfRg",
"host": "opensearch-002",
"transport_address": "10.10.10.2:9300",
"ip": "10.10.10.2",
"name": "opensearch-002"
},
"target": {
"id": "B4XkLHTXRmW7ePYuccoN9g",
"host": "opensearch-003",
"transport_address": "10.10.10.3:9300",
"ip": "10.10.10.3",
"name": "opensearch-003"
},
"index": {
"size": {
"total_in_bytes": 0,
"reused_in_bytes": 0,
"recovered_in_bytes": 0,
"percent": "0.0%"
},
"files": {
"total": 0,
"reused": 0,
"recovered": 0,
"percent": "0.0%"
},
"total_time_in_millis": 0,
"source_throttle_time_in_millis": 0,
"target_throttle_time_in_millis": 0
},
"translog": {
"recovered": 0,
"total": -1,
"percent": "-1.0%",
"total_on_start": -1,
"total_time_in_millis": 0
},
"verify_index": {
"check_index_time_in_millis": 0,
"total_time_in_millis": 0
}
}
]
},
"security-auditlog-2025.12.02": {
"shards": [
{
"id": 0,
"type": "PEER",
"stage": "INIT",
"primary": false,
"start_time_in_millis": 1768231641946,
"total_time_in_millis": 1600736,
"source": {
"id": "-8b2ZEInT16hb9U6KzkfRg",
"host": "opensearch-002",
"transport_address": "10.10.10.2:9300",
"ip": "10.10.10.2",
"name": "opensearch-002"
},
"target": {
"id": "B4XkLHTXRmW7ePYuccoN9g",
"host": "opensearch-003",
"transport_address": "10.10.10.3:9300",
"ip": "10.10.10.3",
"name": "opensearch-003"
},
"index": {
"size": {
"total_in_bytes": 0,
"reused_in_bytes": 0,
"recovered_in_bytes": 0,
"percent": "0.0%"
},
"files": {
"total": 0,
"reused": 0,
"recovered": 0,
"percent": "0.0%"
},
"total_time_in_millis": 0,
"source_throttle_time_in_millis": 0,
"target_throttle_time_in_millis": 0
},
"translog": {
"recovered": 0,
"total": -1,
"percent": "-1.0%",
"total_on_start": -1,
"total_time_in_millis": 0
},
"verify_index": {
"check_index_time_in_millis": 0,
"total_time_in_millis": 0
}
}
]
},
".ds-logstash-MASKED-000011": {
"shards": [
{
"id": 0,
"type": "PEER",
"stage": "INIT",
"primary": false,
"start_time_in_millis": 1768231641973,
"total_time_in_millis": 1600708,
"source": {
"id": "-8b2ZEInT16hb9U6KzkfRg",
"host": "opensearch-002",
"transport_address": "10.10.10.2:9300",
"ip": "10.10.10.2",
"name": "opensearch-002"
},
"target": {
"id": "B4XkLHTXRmW7ePYuccoN9g",
"host": "opensearch-003",
"transport_address": "10.10.10.3:9300",
"ip": "10.10.10.3",
"name": "opensearch-003"
},
"index": {
"size": {
"total_in_bytes": 0,
"reused_in_bytes": 0,
"recovered_in_bytes": 0,
"percent": "0.0%"
},
"files": {
"total": 0,
"reused": 0,
"recovered": 0,
"percent": "0.0%"
},
"total_time_in_millis": 0,
"source_throttle_time_in_millis": 0,
"target_throttle_time_in_millis": 0
},
"translog": {
"recovered": 0,
"total": -1,
"percent": "-1.0%",
"total_on_start": -1,
"total_time_in_millis": 0
},
"verify_index": {
"check_index_time_in_millis": 0,
"total_time_in_millis": 0
}
}
]
},
".ds-logstash-MASKED-000001": {
"shards": [
{
"id": 0,
"type": "PEER",
"stage": "INIT",
"primary": false,
"start_time_in_millis": 1768231651832,
"total_time_in_millis": 1590849,
"source": {
"id": "-8b2ZEInT16hb9U6KzkfRg",
"host": "opensearch-002",
"transport_address": "10.10.10.2:9300",
"ip": "10.10.10.2",
"name": "opensearch-002"
},
"target": {
"id": "B4XkLHTXRmW7ePYuccoN9g",
"host": "opensearch-003",
"transport_address": "10.10.10.3:9300",
"ip": "10.10.10.3",
"name": "opensearch-003"
},
"index": {
"size": {
"total_in_bytes": 0,
"reused_in_bytes": 0,
"recovered_in_bytes": 0,
"percent": "0.0%"
},
"files": {
"total": 0,
"reused": 0,
"recovered": 0,
"percent": "0.0%"
},
"total_time_in_millis": 0,
"source_throttle_time_in_millis": 0,
"target_throttle_time_in_millis": 0
},
"translog": {
"recovered": 0,
"total": -1,
"percent": "-1.0%",
"total_on_start": -1,
"total_time_in_millis": 0
},
"verify_index": {
"check_index_time_in_millis": 0,
"total_time_in_millis": 0
}
}
]
},
"security-auditlog-2025.12.14": {
"shards": [
{
"id": 0,
"type": "PEER",
"stage": "INIT",
"primary": false,
"start_time_in_millis": 1768231642232,
"total_time_in_millis": 1600450,
"source": {
"id": "-8b2ZEInT16hb9U6KzkfRg",
"host": "opensearch-002",
"transport_address": "10.10.10.2:9300",
"ip": "10.10.10.2",
"name": "opensearch-002"
},
"target": {
"id": "B4XkLHTXRmW7ePYuccoN9g",
"host": "opensearch-003",
"transport_address": "10.10.10.3:9300",
"ip": "10.10.10.3",
"name": "opensearch-003"
},
"index": {
"size": {
"total_in_bytes": 0,
"reused_in_bytes": 0,
"recovered_in_bytes": 0,
"percent": "0.0%"
},
"files": {
"total": 0,
"reused": 0,
"recovered": 0,
"percent": "0.0%"
},
"total_time_in_millis": 0,
"source_throttle_time_in_millis": 0,
"target_throttle_time_in_millis": 0
},
"translog": {
"recovered": 0,
"total": -1,
"percent": "-1.0%",
"total_on_start": -1,
"total_time_in_millis": 0
},
"verify_index": {
"check_index_time_in_millis": 0,
"total_time_in_millis": 0
}
}
]
},
"security-auditlog-2025.12.20": {
"shards": [
{
"id": 0,
"type": "PEER",
"stage": "INIT",
"primary": false,
"start_time_in_millis": 1768231646759,
"total_time_in_millis": 1595922,
"source": {
"id": "-8b2ZEInT16hb9U6KzkfRg",
"host": "opensearch-002",
"transport_address": "10.10.10.2:9300",
"ip": "10.10.10.2",
"name": "opensearch-002"
},
"target": {
"id": "B4XkLHTXRmW7ePYuccoN9g",
"host": "opensearch-003",
"transport_address": "10.10.10.3:9300",
"ip": "10.10.10.3",
"name": "opensearch-003"
},
"index": {
"size": {
"total_in_bytes": 0,
"reused_in_bytes": 0,
"recovered_in_bytes": 0,
"percent": "0.0%"
},
"files": {
"total": 0,
"reused": 0,
"recovered": 0,
"percent": "0.0%"
},
"total_time_in_millis": 0,
"source_throttle_time_in_millis": 0,
"target_throttle_time_in_millis": 0
},
"translog": {
"recovered": 0,
"total": -1,
"percent": "-1.0%",
"total_on_start": -1,
"total_time_in_millis": 0
},
"verify_index": {
"check_index_time_in_millis": 0,
"total_time_in_millis": 0
}
}
]
}
}