ISM policy not applying to index

Versions (relevant - OpenSearch/Dashboard/Server OS/Browser):3.7

Describe the issue:ISM Policy is not being applied to data stream

Configuration:

{
  "_id": "insight2",
  "_version": 1,
  "_seq_no": 180323,
  "_primary_term": 2,
  "policy": {
    "policy_id": "insight2",
    "description": "A sample description of the policy",
    "last_updated_time": 1790619351460,
    "schema_version": 29,
    "error_notification": null,
    "default_state": "hot",
    "states": [
      {
        "name": "hot",
        "actions": [
          {
            "retry": {
              "count": 3,
              "backoff": "exponential",
              "delay": "1h"
            },
            "rollover": {
              "min_size": "10gb",
              "min_doc_count": 2000000000,
              "min_index_age": "5d",
              "min_primary_shard_size": "15gb",
              "copy_alias": false
            }
          }
        ],
        "transitions": [
          {
            "state_name": "cold",
            "conditions": {
              "min_index_age": "7d"
            }
          }
        ]
      },
      {
        "name": "cold",
        "actions": [
          {
            "retry": {
              "count": 0,
              "backoff": "exponential",
              "delay": "1h"
            },
            "read_only": {}
          }
        ],
        "transitions": [
          {
            "state_name": "delete",
            "conditions": {
              "min_index_age": "10d"
            }
          }
        ]
      },
      {
        "name": "delete",
        "actions": [
          {
            "retry": {
              "count": 3,
              "backoff": "exponential",
              "delay": "1h"
            },
            "delete": {}
          }
        ],
        "transitions": []
      }
    ],
    "ism_template": [
      {
        "index_patterns": [
          ".ds-insightlogs*"
        ],
        "priority": 1,
        "last_updated_time": 1790619351460
      }
    ]
  }
}

Relevant Logs or Screenshots:

{
  "index_templates": [
    {
      "name": "insightlogs-template",
      "index_template": {
        "index_patterns": [
          "insightlogs*"
        ],
        "template": {
          "settings": {
            "index": {
              "default_pipeline": "insightlogs-pipeline",
              "knn": "true",
              "opendistro": {
                "index_state_management": {
                  "policy_id": "insight2"
                }
              },
              "number_of_shards": "1",
              "number_of_replicas": "0",
              "plugins": {
                "index_state_management": {
                  "policy_id": "insight2"
                }
              }
            }
          },
          "mappings": {
            "properties": {
              "auth-username_vector": {
                "method": {
                  "engine": "lucene",
                  "space_type": "l2",
                  "name": "hnsw"
                },
                "type": "knn_vector",
                "dimension": 384
              },
              "device-name": {
                "type": "keyword"
              },
              "auth-username": {
                "type": "text"
              },
              "cppm-node": {
                "type": "keyword"
              },
              "nas-port": {
                "type": "keyword"
              },
              "nas-id": {
                "type": "keyword"
              },
              "profiles": {
                "type": "keyword"
              },
              "session_id": {
                "type": "keyword"
              },
              "error-code": {
                "type": "integer"
              },
              "mac": {
                "type": "keyword"
              },
              "hostname": {
                "type": "keyword"
              },
              "protocol": {
                "type": "keyword"
              },
              "@timestamp": {
                "type": "date"
              },
              "nas-ip": {
                "type": "ip"
              },
              "auth-method": {
                "type": "keyword"
              },
              "service": {
                "type": "keyword"
              },
              "device-family": {
                "type": "keyword"
              },
              "host": {
                "type": "keyword"
              },
              "device-category": {
                "type": "keyword"
              },
              "timestamp": {
                "format": "yyyy-MM-dd HH:mm:ss.S||yyyy-MM-dd HH:mm:ss||strict_date_optional_time",
                "type": "date"
              },
              "username": {
                "type": "keyword"
              }
            }
          }
        },
        "composed_of": [],
        "priority": 100,
        "_meta": {
          "flow": "simple"
        },
        "data_stream": {
          "timestamp_field": {
            "name": "@timestamp"
          }
        }
      }
    }
  ]
}

Hi @GuyS,

I reproduced this on OpenSearch 3.7.0.

The ism_template pattern .ds-insightlogs* did not result in the data stream backing index being managed by ISM. After changing the pattern in the ISM policy to match the data stream name:

"index_patterns": ["insightlogs*"]

the policy was successfully applied to the newly created backing index.

Could you try changing the pattern from .ds-insightlogs* to insightlogs* and check whether newly created backing indices are managed by ISM?

Kind regards,

Yanlin

Thank you Yanlin !!!

Hi @GuyS,

One additional note regarding the index template configuration: opendistro.index_state_management.policy_id is deprecated in favor of ism_template. The plugins.index_state_management.policy_id field is also deprecated and retained in API responses for backward compatibility. In my tests, both settings were inherited by the data stream backing index, but their presence alone did not confirm that the index was being managed by ISM. You can use the Explain API to verify whether the policy is actually attached:

GET _plugins/_ism/explain/{index}?pretty

Kind regards,

Yanlin