Versions (relevant - OpenSearch/Dashboard/Server OS/Browser):
Opensearch 2.10/2.11 // Opesearch Dashboards 2.10 // Debian 11
Describe the issue:
Some indexes are not rolled over by an ISM policy upon primary shard min size condition. I see that the shards are well over the configured threshold, but _ism/explain shows “Pending rollover…” with outdated size info and nothing happens
Configuration:
{
"_id": "hot_warm_cold_delete",
"_version": 13,
"_seq_no": 37034630,
"_primary_term": 61,
"policy": {
"policy_id": "hot_warm_cold_delete",
"description": "Move older indexes to a warm/cold state/nodes over the time and eventually delete them",
"last_updated_time": 1696931733833,
"schema_version": 19,
"error_notification": null,
"default_state": "hot",
"states": [
{
"name": "hot",
"actions": [
{
"retry": {
"count": 3,
"backoff": "exponential",
"delay": "1m"
},
"allocation": {
"require": {
"temp": "hot"
},
"include": {},
"exclude": {},
"wait_for": false
}
},
{
"retry": {
"count": 3,
"backoff": "exponential",
"delay": "1m"
},
"rollover": {
"min_index_age": "5d",
"min_primary_shard_size": "30gb",
"copy_alias": false
}
}
],
"transitions": [
{
"state_name": "warm",
"conditions": {
"min_index_age": "5d"
}
}
]
},
{
"name": "warm",
"actions": [
{
"retry": {
"count": 3,
"backoff": "exponential",
"delay": "1m"
},
"allocation": {
"require": {
"temp": "warm"
},
"include": {},
"exclude": {},
"wait_for": false
}
},
{
"retry": {
"count": 3,
"backoff": "exponential",
"delay": "1m"
},
"replica_count": {
"number_of_replicas": 1
}
}
],
"transitions": [
{
"state_name": "cold",
"conditions": {
"min_index_age": "30d"
}
}
]
},
{
"name": "cold",
"actions": [
{
"retry": {
"count": 3,
"backoff": "exponential",
"delay": "1m"
},
"allocation": {
"require": {
"temp": "cold"
},
"include": {},
"exclude": {},
"wait_for": false
}
},
{
"retry": {
"count": 3,
"backoff": "exponential",
"delay": "1m"
},
"shrink": {
"num_new_shards": 2
}
},
{
"retry": {
"count": 3,
"backoff": "exponential",
"delay": "1m"
},
"force_merge": {
"max_num_segments": 2
}
}
],
"transitions": [
{
"state_name": "delete",
"conditions": {
"min_index_age": "183d"
}
}
]
},
{
"name": "delete",
"actions": [
{
"retry": {
"count": 3,
"backoff": "exponential",
"delay": "1m"
},
"delete": {}
}
],
"transitions": []
}
],
"ism_template": [
{
"index_patterns": [
"pe.*"
],
"priority": 201,
"last_updated_time": 1686817696462
}
]
}
}
Relevant Logs or Screenshots:
_plugins/_ism/explain/.ds-pe.dir-dovecot-000164
{
".ds-pe.dir-dovecot-000164": {
"index.plugins.index_state_management.policy_id": "hot_warm_cold_delete",
"index.opendistro.index_state_management.policy_id": "hot_warm_cold_delete",
"index": ".ds-pe.dir-dovecot-000164",
"index_uuid": "a5yoXC3DR5CBm82H2rnOew",
"policy_id": "hot_warm_cold_delete",
"policy_seq_no": -2,
"policy_primary_term": 0,
"rolled_over": false,
"index_creation_date": 1698679072350,
"state": {
"name": "hot",
"start_time": 1698679460803
},
"action": {
"name": "rollover",
"start_time": 1698679979151,
"index": 1,
"failed": false,
"consumed_retries": 0,
"last_retry_time": 0
},
"step": {
"name": "attempt_rollover",
"start_time": 1698679979151,
"step_status": "condition_not_met"
},
"retry_info": {
"failed": false,
"consumed_retries": 0
},
"info": {
"message": "Pending rollover of index [index=.ds-pe.dir-dovecot-000164]",
"conditions": {
"min_primary_shard_size": {
"condition": "30gb",
"current": "9gb",
"shard": 1
},
"min_index_age": {
"condition": "5d",
"current": "16h",
"creationDate": 1698679072350
}
}
},
"enabled": true
},
"total_managed_indices": 1
}
but _cat/shards shows different size
# opensearch-cli curl get -P _cat/shards/.ds-pe.dir-dovecot-000164
.ds-pe.dir-dovecot-000164 0 p STARTED 109287785 37.7gb 10.x.x.16 LOGSTORE-HOT-03
.ds-pe.dir-dovecot-000164 0 r STARTED 109287278 37.4gb 10.x.x.14 LOGSTORE-HOT-01
.ds-pe.dir-dovecot-000164 1 p STARTED 109283193 37.6gb 10.x.x.14 LOGSTORE-HOT-01
.ds-pe.dir-dovecot-000164 1 r STARTED 109283704 37.5gb 10.x.x.15 LOGSTORE-HOT-02
.ds-pe.dir-dovecot-000164 2 p STARTED 109285180 37.5gb 10.x.x.15 LOGSTORE-HOT-02
.ds-pe.dir-dovecot-000164 2 r STARTED 109285216 37.4gb 10.x.x.17 LOGSTORE-HOT-04
.ds-pe.dir-dovecot-000164 3 r STARTED 109276464 37.4gb 10.x.x.16 LOGSTORE-HOT-03
.ds-pe.dir-dovecot-000164 3 p STARTED 109276574 37.9gb 10.x.x.17 LOGSTORE-HOT-04