Versions OpenSearch and Dashboard v3.1.0 - Browser Chrome (137.0.7151.69)
Describe the issue: ISM Policies does not take effect in DataStream indexes managed by index Template.
Configuration:
I am running a cluster deployed by OpenSearch Operator. I am ingesting logs from Logstash to OpenSearch cluster with no problems.
I Created a Index Template to assing all DataStream that will be created with a particular index pattern.
The indexes were created and if I check the settings for every one I can confirm that the key
”opendistro.index_state_management.policy_id” is populated with the correct value, but the rollover operation does not occur, and I dont know if this is a bug, because when I call the explain API for ISM policy the result for a DataStream indexes is null.
This is very confusing, because in the settings API call for the index the value is returned.
Relevant Logs or Screenshots:
Policie : my-ism-policy (Rollover when 1mb is reached)
{
"id": "my-ism-policy",
"seqNo": 11747,
"primaryTerm": 2,
"policy": {
"policy_id": "my-ism-policy",
"description": "Rollover para os indices app-logs* com 1mb",
"last_updated_time": 1754330739631,
"schema_version": 24,
"error_notification": null,
"default_state": "hot",
"states": [
{
"name": "hot",
"actions": [
{
"retry": {
"count": 3,
"backoff": "exponential",
"delay": "1m"
},
"rollover": {
"min_size": "1mb",
"copy_alias": false
}
}
],
"transitions": []
}
],
"ism_template": null
}
}
Index Templates: my-data-stream-template and my-data-stream-template-3 ( I tried the first one using opendistro.index_state_management.policy_id and the other one using plugins.index_state_management.policy_id)
GET _index_template/my-data-stream-template
{
"index_templates": [
{
"name": "my-data-stream-template",
"index_template": {
"index_patterns": [
"my-data-stream-*"
],
"template": {
"settings": {
"index": {
"opendistro": {
"index_state_management": {
"policy_id": "my-ism-policy"
}
}
}
}
},
"composed_of": [],
"priority": 100,
"data_stream": {
"timestamp_field": {
"name": "@timestamp"
}
}
}
}
]
}
GET _index_template/my-data-stream-template-3
{
"index_templates": [
{
"name": "my-data-stream-template-3",
"index_template": {
"index_patterns": [
"my-data-stream-3*"
],
"template": {
"settings": {
"index": {
"plugins": {
"index_state_management": {
"policy_id": "my-ism-policy"
}
}
}
}
},
"composed_of": [],
"priority": 90,
"data_stream": {
"timestamp_field": {
"name": "@timestamp"
}
}
}
}
]
}
Index Settings: Showing that the ism policy was inherited from index template
GET .ds-my-data-stream-3-app-logs-hml-integradorproducao-000001/_settings
{
".ds-my-data-stream-3-app-logs-hml-integradorproducao-000001": {
"settings": {
"index": {
"replication": {
"type": "DOCUMENT"
},
"opendistro": {
"index_state_management": {
"policy_id": "my-ism-policy"
}
},
"hidden": "true",
"number_of_shards": "1",
"provided_name": ".ds-my-data-stream-3-app-logs-hml-integradorproducao-000001",
"creation_date": "1754338757630",
"number_of_replicas": "1",
"uuid": "G_Bk0zsiSW23uspWgWX5Nw",
"version": {
"created": "137227827"
}
}
}
}
}
Explain: Showing that ism policy is null
GET _plugins/_ism/explain/.ds-my-data-stream-3-app-logs-hml-integradorproducao-000001
{
".ds-my-data-stream-3-app-logs-hml-integradorproducao-000001": {
"index.plugins.index_state_management.policy_id": null,
"index.opendistro.index_state_management.policy_id": null,
"enabled": null
},
"total_managed_indices": 0
}