ISM doesn't delete old data stream index

Plugins Index Management
1

Versions (relevant - OpenSearch/Dashboard/Server OS/Browser):

  • 2.1

Describe the issue:
The ISM policy for daily rotation and log deletion via cron succeeds on the rollover for data stream, but completely ignores the step to delete the old index.
After all the steps explain API shows that everything has passed successfully, but the old index is in Closed status, but it has not been deleted.

Configuration:

{
    "id": "rw_rollover_delete",
    "seqNo": 3551056,
    "primaryTerm": 12,
    "policy": {
        "policy_id": "rw_rollover_delete",
        "description": "Daily cron rotation",
        "last_updated_time": 1691062620000,
        "schema_version": 16,
        "error_notification": null,
        "default_state": "init",
        "states": [
            {
                "name": "init",
                "actions": [
                    {
                        "timeout": "20m",
                        "retry": {
                            "count": 10,
                            "backoff": "exponential",
                            "delay": "5m"
                        },
                        "read_write": {}
                    }
                ],
                "transitions": [
                    {
                        "state_name": "rollover",
                        "conditions": {
                            "cron": {
                                "cron": {
                                    "expression": "0 0 * * *",
                                    "timezone": "Europe/Sofia"
                                }
                            }
                        }
                    }
                ]
            },
            {
                "name": "rollover",
                "actions": [
                    {
                        "timeout": "1h",
                        "retry": {
                            "count": 10,
                            "backoff": "exponential",
                            "delay": "10m"
                        },
                        "rollover": {
                            "min_doc_count": 1
                        }
                    }
                ],
                "transitions": []
            },
            {
                "name": "delete",
                "actions": [
                    {
                        "retry": {
                            "count": 3,
                            "backoff": "exponential",
                            "delay": "1m"
                        },
                        "delete": {}
                    }
                ],
                "transitions": []
            }
        ],
        "ism_template": [
            {
                "index_patterns": [
                    ".ds-some-pattern"
                ],
                "priority": 1,
                "last_updated_time": 169106262000
            },
            {
                "index_patterns": [
                    "some-pattern"
                ],
                "priority": 1,
                "last_updated_time": 169106262000
            }
        ]
    }
}

Relevant Logs or Screenshots:

2

Hi @paragon21. Your ISM does not delete an index because you have not sent it for deletion. So I added a transition to delete. Note that the index name and other settings are not from your example.
But tell me, do you use this ISM for a datastream or a regular index?

{
    "id": "asdasd",
    "seqNo": 1402728,
    "primaryTerm": 21,
    "policy": {
        "policy_id": "asdasd",
        "description": "A simple default policy that changes the replica count between hot and cold states.",
        "last_updated_time": 1691066704384,
        "schema_version": 18,
        "error_notification": null,
        "default_state": "init",
        "states": [
            {
                "name": "init",
                "actions": [
                    {
                        "timeout": "20m",
                        "retry": {
                            "count": 10,
                            "backoff": "exponential",
                            "delay": "5m"
                        },
                        "read_write": {}
                    }
                ],
                "transitions": [
                    {
                        "state_name": "rollover",
                        "conditions": {
                            "cron": {
                                "cron": {
                                    "expression": "0 0 * * *",
                                    "timezone": "Europe/Sofia"
                                }
                            }
                        }
                    }
                ]
            },
            {
                "name": "rollover",
                "actions": [
                    {
                        "timeout": "1h",
                        "retry": {
                            "count": 10,
                            "backoff": "exponential",
                            "delay": "10m"
                        },
                        "rollover": {
                            "min_doc_count": 1
                        }
                    }
                ],
                "transitions": [
                    {
                        "state_name": "delete",
                        "conditions": {
                            "min_rollover_age": "1d"
                        }
                    }
                ]
            },
            {
                "name": "delete",
                "actions": [
                    {
                        "retry": {
                            "count": 3,
                            "backoff": "exponential",
                            "delay": "1m"
                        },
                        "delete": {}
                    }
                ],
                "transitions": []
            }
        ],
        "ism_template": [
            {
                "index_patterns": [
                    "example-index-*"
                ],
                "priority": 0,
                "last_updated_time": 1691066624301
            }
        ]
    }
}

image
image578Ă—749 27.7 KB

1 Like
3

Hi, @Kruzerson! Thank you very much, I’ll try that tonight. I thought that if I specifying the order, the sequence of execution would be guaranteed, and that specifying transition is optional.
I use this policy to rollover data_streams, I decided not to use regular indexes because I have a large number of logically separated indexes and occasional errors of linking an alias to a new index after rollover.
I liked data-streams as a more “smart” abstraction that does not require aliases and rollover aliases.

4

@paragon21 In this case, I recommend using the following IMS for data stream.

{
    "id": "asdasd",
    "seqNo": 1424250,
    "primaryTerm": 21,
    "policy": {
        "policy_id": "asdasd",
        "description": "A simple default policy that changes the replica count between hot and cold states.",
        "last_updated_time": 1691146376673,
        "schema_version": 18,
        "error_notification": null,
        "default_state": "init",
        "states": [
            {
                "name": "init",
                "actions": [
                    {
                        "retry": {
                            "count": 3,
                            "backoff": "exponential",
                            "delay": "1m"
                        },
                        "rollover": {
                            "min_index_age": "1d"
                        }
                    }
                ],
                "transitions": [
                    {
                        "state_name": "delete",
                        "conditions": {
                            "cron": {
                                "cron": {
                                    "expression": "0 0 * * *",
                                    "timezone": "Europe/Sofia"
                                }
                            }
                        }
                    }
                ]
            },
            {
                "name": "delete",
                "actions": [
                    {
                        "retry": {
                            "count": 3,
                            "backoff": "exponential",
                            "delay": "1m"
                        },
                        "delete": {}
                    }
                ],
                "transitions": []
            }
        ],
        "ism_template": [
            {
                "index_patterns": [
                    "example-index-*"
                ],
                "priority": 0,
                "last_updated_time": 1691066624301
            }
        ]
    }
}
5

This topic was automatically closed 60 days after the last reply. New replies are no longer allowed.