Versions :
OpenSearch 2.14 / Ubuntu 22.04.4 LTS / Google Chrome 127.0.6533.72
Describe the issue:
A server was shut down for a move that took a couple of weeks. After firing it up again, we have two red shards in the OpenSearch cluster but not sure how that happened. .opendistro-job-scheduler-lock and .opendistro-ism-config are red
We get a [search_phase_execution_exception] all shards failed when trying to view state managed policies or policy managed indexes.
We tried to restart the node to see if it would change anything, it doesn’t.
We wonder if we can recover easily from this or should resort to a backup instead.
Thanks!
Configuration:
Single node
Relevant Logs or Screenshots:
GET _cluster/health?level=shards
{
“cluster_name”: “opensearch”,
“status”: “red”,
“timed_out”: false,
“number_of_nodes”: 1,
“number_of_data_nodes”: 1,
“discovered_master”: true,
“discovered_cluster_manager”: true,
“active_primary_shards”: 293,
“active_shards”: 293,
“relocating_shards”: 0,
“initializing_shards”: 0,
“unassigned_shards”: 19,
“delayed_unassigned_shards”: 0,
“number_of_pending_tasks”: 0,
“number_of_in_flight_fetch”: 0,
“task_max_waiting_in_queue_millis”: 0,
“active_shards_percent_as_number”: 93.91025641025641
}
…
".opendistro-job-scheduler-lock": {
"status": "red",
"number_of_shards": 1,
"number_of_replicas": 0,
"active_primary_shards": 0,
"active_shards": 0,
"relocating_shards": 0,
"initializing_shards": 0,
"unassigned_shards": 1,
"shards": {
"0": {
"status": "red",
"primary_active": false,
"active_shards": 0,
"relocating_shards": 0,
"initializing_shards": 0,
"unassigned_shards": 1
}
}
},
…
".opendistro-ism-config": {
"status": "red",
"number_of_shards": 1,
"number_of_replicas": 0,
"active_primary_shards": 0,
"active_shards": 0,
"relocating_shards": 0,
"initializing_shards": 0,
"unassigned_shards": 1,
"shards": {
"0": {
"status": "red",
"primary_active": false,
"active_shards": 0,
"relocating_shards": 0,
"initializing_shards": 0,
"unassigned_shards": 1
}
}
},
…
GET _cluster/allocation/explain?pretty
**{ **
** “index”: “.opendistro-ism-config”, **
** “shard”: 0, **
** “primary”: true **
}
{
“index”: “.opendistro-ism-config”,
“shard”: 0,
“primary”: true,
“current_state”: “unassigned”,
“unassigned_info”: {
“reason”: “ALLOCATION_FAILED”,
“at”: “2024-08-05T15:30:20.763Z”,
“failed_allocation_attempts”: 5,
“details”: "failed shard on node [rOmvdDc6S223rdaLtiCbDw]: failed recovery, failure RecoveryFailedException[[.opendistro-ism-config][0]: Recovery failed on {blabla.com}{rOmvdDc6S223rdaLtiCbDw}{m1UhiXs3SPimXGaq7hfHJQ}{someip}{someip:someport}{dimr}{shard_indexing_pressure_enabled=true}]; nested: IndexShardRecoveryException[failed recovery]; nested: TranslogCorruptedException[translog from source [/var/lib/opensearch/nodes/0/indices/1iR6Oo9EQX6F79AVY1IyIw/0/translog] is corrupted]; nested: NoSuchFileException[/var/lib/opensearch/nodes/0/indices/1iR6Oo9EQX6F79AVY1IyIw/0/translog/translog-3341053.tlog]; ",
“last_allocation_status”: “no”
},
“can_allocate”: “no”,
“allocate_explanation”: “cannot allocate because allocation is not permitted to any of the nodes that hold an in-sync shard copy”,
“node_allocation_decisions”: [
{
“node_id”: “rOmvdDc6S223rdaLtiCbDw”,
“node_name”: “blabla.com”,
“transport_address”: “someip:someport”,
“node_attributes”: {
“remote_store.repository.opensearch_nfs.settings.location”: “/mnt/opensearch_nfs”,
“remote_store.repository.opensearch_nfs.type”: “fs”,
“remote_store.state.repository”: “opensearch_nfs”,
“shard_indexing_pressure_enabled”: “true”,
“remote_store.segment.repository”: “opensearch_nfs”,
“remote_store.translog.repository”: “opensearch_nfs”
},
“node_decision”: “no”,
“store”: {
“in_sync”: true,
“allocation_id”: “lTo39lSOR-qPVXcY3fL1sA”
},
“deciders”: [
{
“decider”: “max_retry”,
“decision”: “NO”,
“explanation”: “shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2024-08-05T15:30:20.763Z], failed_attempts[5], failed_nodes[[rOmvdDc6S223rdaLtiCbDw]], delayed=false, details[failed shard on node [rOmvdDc6S223rdaLtiCbDw]: failed recovery, failure RecoveryFailedException[[.opendistro-ism-config][0]: Recovery failed on {blabla.com}{rOmvdDc6S223rdaLtiCbDw}{m1UhiXs3SPimXGaq7hfHJQ}{someip}{someip:someport}{dimr}{shard_indexing_pressure_enabled=true}]; nested: IndexShardRecoveryException[failed recovery]; nested: TranslogCorruptedException[translog from source [/var/lib/opensearch/nodes/0/indices/1iR6Oo9EQX6F79AVY1IyIw/0/translog] is corrupted]; nested: NoSuchFileException[/var/lib/opensearch/nodes/0/indices/1iR6Oo9EQX6F79AVY1IyIw/0/translog/translog-3341053.tlog]; ], allocation_status[deciders_no]]]”
}
]
}
]
}
GET _cluster/allocation/explain?pretty
**{ **
** “index”: “.opendistro-job-scheduler-lock”, **
** “shard”: 0, **
** “primary”: true **
}
{
“index”: “.opendistro-job-scheduler-lock”,
“shard”: 0,
“primary”: true,
“current_state”: “unassigned”,
“unassigned_info”: {
“reason”: “ALLOCATION_FAILED”,
“at”: “2024-08-05T15:30:20.761Z”,
“failed_allocation_attempts”: 5,
“details”: "failed shard on node [rOmvdDc6S223rdaLtiCbDw]: failed recovery, failure RecoveryFailedException[[.opendistro-job-scheduler-lock][0]: Recovery failed on {blabla.com}{rOmvdDc6S223rdaLtiCbDw}{m1UhiXs3SPimXGaq7hfHJQ}{someip}{someip:someport}{dimr}{shard_indexing_pressure_enabled=true}]; nested: IndexShardRecoveryException[failed recovery]; nested: TranslogCorruptedException[translog from source [/var/lib/opensearch/nodes/0/indices/upwVvFxLTtiz6zX4D0BE6Q/0/translog/translog-3455355.tlog] is corrupted, translog header truncated]; nested: EOFException; ",
“last_allocation_status”: “no”
},
“can_allocate”: “no”,
“allocate_explanation”: “cannot allocate because allocation is not permitted to any of the nodes that hold an in-sync shard copy”,
“node_allocation_decisions”: [
{
“node_id”: “rOmvdDc6S223rdaLtiCbDw”,
“node_name”: “blabla.com”,
“transport_address”: “someip:someport”,
“node_attributes”: {
“remote_store.repository.opensearch_nfs.settings.location”: “/mnt/opensearch_nfs”,
“remote_store.repository.opensearch_nfs.type”: “fs”,
“remote_store.state.repository”: “opensearch_nfs”,
“shard_indexing_pressure_enabled”: “true”,
“remote_store.segment.repository”: “opensearch_nfs”,
“remote_store.translog.repository”: “opensearch_nfs”
},
“node_decision”: “no”,
“store”: {
“in_sync”: true,
“allocation_id”: “lnGpK294T9CpBSczUkUaBQ”
},
“deciders”: [
{
“decider”: “max_retry”,
“decision”: “NO”,
“explanation”: “shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2024-08-05T15:30:20.761Z], failed_attempts[5], failed_nodes[[rOmvdDc6S223rdaLtiCbDw]], delayed=false, details[failed shard on node [rOmvdDc6S223rdaLtiCbDw]: failed recovery, failure RecoveryFailedException[[.opendistro-job-scheduler-lock][0]: Recovery failed on {blabla.com}{rOmvdDc6S223rdaLtiCbDw}{m1UhiXs3SPimXGaq7hfHJQ}{someip}{someip:someport}{dimr}{shard_indexing_pressure_enabled=true}]; nested: IndexShardRecoveryException[failed recovery]; nested: TranslogCorruptedException[translog from source [/var/lib/opensearch/nodes/0/indices/upwVvFxLTtiz6zX4D0BE6Q/0/translog/translog-3455355.tlog] is corrupted, translog header truncated]; nested: EOFException; ], allocation_status[deciders_no]]]”
}
]
}
]
}