Index state management abnormal executed

OpenSearch
1

Versions (relevant - OpenSearch/Dashboard/Server OS/Browser):

{
  "name" : "node-1",
  "cluster_name" : "lsmsp-opensearch",
  "cluster_uuid" : "MshAYjFJS9GGYaOpmQ8P8Q",
  "version" : {
    "distribution" : "opensearch",
    "number" : "1.3.5",
    "build_type" : "rpm",
    "build_hash" : "6cabc6aacc030bcaab30aeacf7529bbe7415c61b",
    "build_date" : "2022-08-30T17:39:57.094896Z",
    "build_snapshot" : false,
    "lucene_version" : "8.10.1",
    "minimum_wire_compatibility_version" : "6.8.0",
    "minimum_index_compatibility_version" : "6.0.0-beta1"
  },
  "tagline" : "The OpenSearch Project: https://opensearch.org/"
}

Describe the issue:

Index state management abnormal executed!
They were matched policy and rollover successfully, but abnormal deleted by policy.
Its weird, why they were deleted in one day?

Configuration:

curl -k -X GET -H 'Content-Type: application/json' http://localhost:9248/_plugins/_ism/policies/eventlog_policy?pretty
{
"_id" : "eventlog_policy",
"_version" : 2,
"_seq_no" : 14386964,
"_primary_term" : 2,
"policy" : {
"policy_id" : "eventlog_policy",
"description" : "my default policy",
"last_updated_time" : 1717036466857,
"schema_version" : 13,
"error_notification" : null,
"default_state" : "hot",
"states" : [
{
"name" : "hot",
"actions" : [
{
"retry" : {
"count" : 3,
"backoff" : "exponential",
"delay" : "1m"
},
"rollover" : {
"min_index_age" : "1d"
}
}
],
"transitions" : [
{
"state_name" : "warm"
}
]
},
{
"name" : "warm",
"actions" : [ ],
"transitions" : [
{
"state_name" : "delete",
"conditions" : {
"min_index_age" : "200d"
}
}
]
},
{
"name" : "delete",
"actions" : [
{
"retry" : {
"count" : 3,
"backoff" : "exponential",
"delay" : "1m"
},
"delete" : { }
}
],
"transitions" : [ ]
}
],
"ism_template" : [
{
"index_patterns" : [
"eventlog"
],
"priority" : 100,
"last_updated_time" : 1717036466857
}
]
}
}

Relevant Logs or Screenshots:

lsmsp-opensearch-2024-04-13-1.log:108575:[2024-04-13T20:11:04,612][INFO ][o.o.i.i.s.r.AttemptRolloverStep] [node-1] .ds-eventlog-000152 rollover conditions evaluated to true [indexCreationDate=1712923693810, numDocs=993603, indexSize=735270278, primaryShardSize=735270278]
lsmsp-opensearch-2024-04-14-1.log:109470:[2024-04-14T20:12:23,922][INFO ][o.o.i.i.s.r.AttemptRolloverStep] [node-1] .ds-eventlog-000153 rollover conditions evaluated to true [indexCreationDate=1713010264655, numDocs=928334, indexSize=694484354, primaryShardSize=694484354]
lsmsp-opensearch-2024-04-15-1.log:110243:[2024-04-15T20:12:42,700][INFO ][o.o.i.i.s.r.AttemptRolloverStep] [node-1] .ds-eventlog-000154 rollover conditions evaluated to true [indexCreationDate=1713096743925, numDocs=1862472, indexSize=1380521379, primaryShardSize=1380521379]
lsmsp-opensearch-2024-04-16-1.log:110966:[2024-04-16T20:12:55,751][INFO ][o.o.i.i.s.r.AttemptRolloverStep] [node-1] .ds-eventlog-000155 rollover conditions evaluated to true [indexCreationDate=1713183162732, numDocs=1870311, indexSize=1398176578, primaryShardSize=1398176578]
lsmsp-opensearch-2024-04-17-1.log:111792:[2024-04-17T20:14:13,651][INFO ][o.o.i.i.s.r.AttemptRolloverStep] [node-1] .ds-eventlog-000156 rollover conditions evaluated to true [indexCreationDate=1713269575773, numDocs=1862235, indexSize=1387528575, primaryShardSize=1387528575]
lsmsp-opensearch-2024-04-18-1.log:112521:[2024-04-18T20:14:36,268][INFO ][o.o.i.i.s.r.AttemptRolloverStep] [node-1] .ds-eventlog-000157 rollover conditions evaluated to true [indexCreationDate=1713356053723, numDocs=1904549, indexSize=1419679670, primaryShardSize=1419679670]
lsmsp-opensearch-2024-04-19-1.log:113308:[2024-04-19T20:14:58,039][INFO ][o.o.i.i.s.r.AttemptRolloverStep] [node-1] .ds-eventlog-000158 rollover conditions evaluated to true [indexCreationDate=1713442476280, numDocs=1868592, indexSize=1386995212, primaryShardSize=1386995212]
lsmsp-opensearch-2024-04-20-1.log:114064:[2024-04-20T20:15:16,216][INFO ][o.o.i.i.s.r.AttemptRolloverStep] [node-1] .ds-eventlog-000159 rollover conditions evaluated to true [indexCreationDate=1713528898058, numDocs=1154524, indexSize=857423745, primaryShardSize=857423745]
lsmsp-opensearch-2024-04-21-1.log:102836:[2024-04-21T20:15:32,097][INFO ][o.o.i.i.s.r.AttemptRolloverStep] [node-1] .ds-eventlog-000160 rollover conditions evaluated to true [indexCreationDate=1713615316235, numDocs=904572, indexSize=678240547, primaryShardSize=678240547]
lsmsp-opensearch-2024-04-22-1.log:64589:[2024-04-22T20:17:05,198][INFO ][o.o.i.i.s.r.AttemptRolloverStep] [node-1] .ds-eventlog-000161 rollover conditions evaluated to true [indexCreationDate=1713701732100, numDocs=1815219, indexSize=1345010946, primaryShardSize=1345010946]
lsmsp-opensearch-2024-04-23-1.log:65402:[2024-04-23T20:18:39,662][INFO ][o.o.i.i.s.r.AttemptRolloverStep] [node-1] .ds-eventlog-000162 rollover conditions evaluated to true [indexCreationDate=1713788225215, numDocs=1841711, indexSize=1383461055, primaryShardSize=1383461055]
lsmsp-opensearch-2024-04-24-1.log:66179:[2024-04-24T20:19:37,762][INFO ][o.o.i.i.s.r.AttemptRolloverStep] [node-1] .ds-eventlog-000163 rollover conditions evaluated to true [indexCreationDate=1713874719679, numDocs=1833783, indexSize=1377165961, primaryShardSize=1377165961]
lsmsp-opensearch-2024-04-25-1.log:67085:[2024-04-25T20:22:32,146][INFO ][o.o.i.i.s.r.AttemptRolloverStep] [node-1] .ds-eventlog-000164 rollover conditions evaluated to true [indexCreationDate=1713961177778, numDocs=1852683, indexSize=1372374171, primaryShardSize=1372374171]


lsmsp-opensearch-2024-04-21-1.json:87513:{"type": "server", "timestamp": "2024-04-21T15:25:21,135+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000090/7bD9PwHiSIKMUxmX2JL1pA] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87516:{"type": "server", "timestamp": "2024-04-21T15:25:21,239+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000091/2zKpFrTcTSOU3mF8uKzfsg] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87520:{"type": "server", "timestamp": "2024-04-21T15:25:21,351+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000092/9KnjIyKYSzayIMRdL-Lkuw] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87525:{"type": "server", "timestamp": "2024-04-21T15:25:21,447+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000093/TeiM75G0RIqBTTpVJMWjHQ] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87528:{"type": "server", "timestamp": "2024-04-21T15:25:21,501+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000094/GZh7RAJuSNOY04MoUhH-XA] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87531:{"type": "server", "timestamp": "2024-04-21T15:25:21,563+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000095/L-hqIw3sQCKhc922wF51tA] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87534:{"type": "server", "timestamp": "2024-04-21T15:25:21,623+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000096/BDptaaGRSEa-PkeGealPLw] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87537:{"type": "server", "timestamp": "2024-04-21T15:25:21,748+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000097/6ITFKKlKQYKg-2Qyskm3tg] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87540:{"type": "server", "timestamp": "2024-04-21T15:25:21,869+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000098/FuftKEeJQ12RfmV40rhD8A] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87543:{"type": "server", "timestamp": "2024-04-21T15:25:21,981+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000099/AwQN020AQrun5Bj-WfrOWA] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87546:{"type": "server", "timestamp": "2024-04-21T15:25:22,083+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000100/faFK5n1_RwqB45_-CCO0BQ] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87550:{"type": "server", "timestamp": "2024-04-21T15:25:22,188+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000101/PTJIMxzwSha_wTtqZrXPHA] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87553:{"type": "server", "timestamp": "2024-04-21T15:25:22,322+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000102/AWpBLx0sTuC9vVAD5eR8-g] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87558:{"type": "server", "timestamp": "2024-04-21T15:25:22,486+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000103/0dxNOVGOSb6g5bmzpm_N-g] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87561:{"type": "server", "timestamp": "2024-04-21T15:25:22,578+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000104/k4AFTmGaROe-MHol_Znsaw] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87565:{"type": "server", "timestamp": "2024-04-21T15:25:22,639+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000105/T-VOPcaLT0ewYdSb0GJhYw] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87568:{"type": "server", "timestamp": "2024-04-21T15:25:22,733+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000106/CQp0jl2VTiSjxlmYz2aXAg] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87571:{"type": "server", "timestamp": "2024-04-21T15:25:22,858+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000107/6EyDLl3TSJaU6bl_jebTyQ] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87574:{"type": "server", "timestamp": "2024-04-21T15:25:22,968+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000108/xtFl8Y-HQVSVJoPjxOyONg] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87577:{"type": "server", "timestamp": "2024-04-21T15:25:23,121+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000109/XyxdRWC_SLm5E5RrUNy7Wg] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87580:{"type": "server", "timestamp": "2024-04-21T15:25:23,296+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000110/LrYkYbrRQj65recG2uzhPQ] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87583:{"type": "server", "timestamp": "2024-04-21T15:25:23,424+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000111/BRPT6NNSTrSPXZhc0E4VlA] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87588:{"type": "server", "timestamp": "2024-04-21T15:25:23,558+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000112/2QEylPp4SqG1z1I2HpPXdQ] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87591:{"type": "server", "timestamp": "2024-04-21T15:25:23,671+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000113/NupGk9jWS_e2SeQ20_T3xQ] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87594:{"type": "server", "timestamp": "2024-04-21T15:25:23,807+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000114/srOBz7bMS_ivN51iBt6MgQ] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87597:{"type": "server", "timestamp": "2024-04-21T15:25:23,963+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000115/b7sjmuRRRhiUErmU54D0xg] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87600:{"type": "server", "timestamp": "2024-04-21T15:25:24,093+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000116/nCFK5NgDT8esc_JOAW10-w] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87604:{"type": "server", "timestamp": "2024-04-21T15:25:24,256+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000117/7KsksDoGTBK8YfnnKgwcoA] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87607:{"type": "server", "timestamp": "2024-04-21T15:25:24,369+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000118/-yX78wg2TSeUg1LUSVz_Gg] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87610:{"type": "server", "timestamp": "2024-04-21T15:25:24,508+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000119/rmLlPMllQ3y2cpLqWcZghQ] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87615:{"type": "server", "timestamp": "2024-04-21T15:25:24,656+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000120/0DSlusqKQCGPCkjokaVyXg] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87618:{"type": "server", "timestamp": "2024-04-21T15:25:24,773+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000121/DTqDc6oJRtysEG8_WJsTxw] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87621:{"type": "server", "timestamp": "2024-04-21T15:25:24,848+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000122/lvKOFoEZSoq1h4ih6WMjEA] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87624:{"type": "server", "timestamp": "2024-04-21T15:25:24,949+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000123/6HnhnUrrR7uZFFgKoESJ0Q] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87627:{"type": "server", "timestamp": "2024-04-21T15:25:25,069+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000124/t9be7CjiSSePR_6YdAsllg] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87630:{"type": "server", "timestamp": "2024-04-21T15:25:25,151+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000125/cPW7dk40ReGP_O8bd_Q_GQ] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87633:{"type": "server", "timestamp": "2024-04-21T15:25:25,222+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000126/AMObNPJMQTuN55OANuy9qA] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87636:{"type": "server", "timestamp": "2024-04-21T15:25:25,339+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000127/cJZ_YkvcS-KqWFVNOOpfqA] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87411:{"type": "server", "timestamp": "2024-04-21T15:25:15,834+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000128/AhzmdjdKRXmDjfIb9q54dA] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87415:{"type": "server", "timestamp": "2024-04-21T15:25:16,194+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000129/pBWQbVrNTs2BLa9XsdjlFg] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87420:{"type": "server", "timestamp": "2024-04-21T15:25:16,499+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000130/uKLliC2kQW6f4CpduGPxXQ] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87423:{"type": "server", "timestamp": "2024-04-21T15:25:16,708+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000131/gGoocguzTkOUTdHgd3V_6Q] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87426:{"type": "server", "timestamp": "2024-04-21T15:25:16,846+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000132/KFIGM_3mQyG3h3XqlAFN8w] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87429:{"type": "server", "timestamp": "2024-04-21T15:25:16,963+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000133/NweUDnDeT4a79Dlqzc-3Lw] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87432:{"type": "server", "timestamp": "2024-04-21T15:25:17,133+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000134/t4SEEOViTKudT2byheHMOg] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87437:{"type": "server", "timestamp": "2024-04-21T15:25:17,332+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000135/Wh-k0TpsTryKBKwvHqg3yQ] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87440:{"type": "server", "timestamp": "2024-04-21T15:25:17,530+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000136/UzuxcABkRZeuRa46hsDvSA] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87443:{"type": "server", "timestamp": "2024-04-21T15:25:17,697+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000137/yEqJzL-JTG-4_7ttxl-0Iw] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87446:{"type": "server", "timestamp": "2024-04-21T15:25:17,819+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000138/SHo33BScTimr_nfhX-MVnw] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87449:{"type": "server", "timestamp": "2024-04-21T15:25:17,920+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000139/l39bgecyQTWeTJa2kgaFwA] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87452:{"type": "server", "timestamp": "2024-04-21T15:25:18,011+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000140/G8fVQhBBQcKOKqQTFPFRlQ] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87455:{"type": "server", "timestamp": "2024-04-21T15:25:18,157+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000141/vYO6m69LRK-S5pW9cvWX4g] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87456:{"type": "server", "timestamp": "2024-04-21T15:25:18,242+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000142/LjjADUtuT5Kywx9Byf9rKw] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87461:{"type": "server", "timestamp": "2024-04-21T15:25:18,407+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000143/jErpm7MUSwqBYSJOEsMUCA] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87464:{"type": "server", "timestamp": "2024-04-21T15:25:18,501+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000144/MzC95K9KQsqDREINPl3cVQ] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87467:{"type": "server", "timestamp": "2024-04-21T15:25:18,584+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000145/IfG4PMzZQ_mBeXmoClSH6w] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87470:{"type": "server", "timestamp": "2024-04-21T15:25:18,703+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000146/RQ_isDB7TnucyovUJmB68g] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87473:{"type": "server", "timestamp": "2024-04-21T15:25:18,798+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000147/d1eVIbTTShaMxlgxTUCh9Q] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87476:{"type": "server", "timestamp": "2024-04-21T15:25:18,936+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000148/SvUXMbXiSROhoBwOp7-REQ] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87479:{"type": "server", "timestamp": "2024-04-21T15:25:19,094+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000149/XTFjd5WfQCG4EBqO7M6jYQ] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87482:{"type": "server", "timestamp": "2024-04-21T15:25:19,231+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000150/zzvSwMZOQSGivlVXVzHXww] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87483:{"type": "server", "timestamp": "2024-04-21T15:25:19,409+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000151/Jw91BkZcSuGLCgvhmU6yjQ] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87486:{"type": "server", "timestamp": "2024-04-21T15:25:19,600+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000152/ijNb4MeBSZS46jjJXhi-gQ] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87491:{"type": "server", "timestamp": "2024-04-21T15:25:19,696+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000153/mlcR1nvwTL2_kNRMZFV2VQ] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87494:{"type": "server", "timestamp": "2024-04-21T15:25:19,920+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000154/5fw4H0lARXyIZ9O5v74JRg] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87497:{"type": "server", "timestamp": "2024-04-21T15:25:20,058+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000155/u2dpFpyST7mOtymfKyk6Bg] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87500:{"type": "server", "timestamp": "2024-04-21T15:25:20,263+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000156/n-nfSfvGQ3a2nj_3Li9wEw] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87503:{"type": "server", "timestamp": "2024-04-21T15:25:20,546+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000157/0keaXl5aShule3k6adWofA] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87506:{"type": "server", "timestamp": "2024-04-21T15:25:20,710+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000158/_Bt2k-ZvT9-YGewgN9NpHA] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
lsmsp-opensearch-2024-04-21-1.json:87510:{"type": "server", "timestamp": "2024-04-21T15:25:20,983+08:00", "level": "INFO", "component": "o.o.c.m.MetadataDeleteIndexService", "cluster.name": "lsmsp-opensearch", "node.name": "node-1", "message": "[.ds-eventlog-000159/Lpq8Jh0PQjmZrdjZFcl7YQ] deleting index", "cluster.uuid": "MshAYjFJS9GGYaOpmQ8P8Q", "node.id": "jTxWWId3Q96wnTiX3M4arw"  }
2

This states that policy was updated yesterday. What was the previous configuration?

3

last version of this policy warm state is 180d，will send here later

4

hi~@pablo, its my previous config about eventlog_policy

{
      "_id" : "ccp_default_policy",
      "_seq_no" : 0,
      "_primary_term" : 1,
      "policy" : {
        "policy_id" : "ccp_default_policy",
        "description" : "CCP default policy",
        "last_updated_time" : 1699862083580,
        "schema_version" : 13,
        "error_notification" : null,
        "default_state" : "hot",
        "states" : [
          {
            "name" : "hot",
            "actions" : [
              {
                "retry" : {
                  "count" : 3,
                  "backoff" : "exponential",
                  "delay" : "1m"
                },
                "rollover" : {
                  "min_index_age" : "1d"
                }
              }
            ],
            "transitions" : [
              {
                "state_name" : "warm"
              }
            ]
          },
          {
            "name" : "warm",
            "actions" : [ ],
            "transitions" : [
              {
                "state_name" : "delete",
                "conditions" : {
                  "min_index_age" : "180d"
                }
              }
            ]
          },
          {
            "name" : "delete",
            "actions" : [
              {
                "retry" : {
                  "count" : 3,
                  "backoff" : "exponential",
                  "delay" : "1m"
                },
                "delete" : { }
              }
            ],
            "transitions" : [ ]
          }
        ],
        "ism_template" : null
      }
    }
5

Hi！
if i change policy to a data stream, will the indeices of data stream become same lifecycle state?
for example:

before policy: 1days: hot->warm 5days warm->cold 1days code->delete

.ds-log-000001,.ds-log-000002,.ds-log-000003,.ds-log-000004 current state were warm

then i change policy to
after policy: 1days: hot->warm 10days warm->cold 1days code->delete

will these 4 indices will come to warm same time？（after 10 days）