Versions (relevant - OpenSearch/Dashboard/Server OS/Browser):
Opensearch 2.19
Describe the issue:
I have a bucket-level monitor in OpenSearch. When I run a dry run, the monitor triggers correctly. However, on scheduled runs:
- The monitor input returns buckets correctly.
- Trigger condition is set to true (for debugging).
- No actions are fired, and action_results is empty.
I have tried multiple action configurations: Per execution, per alert (dedupe, new, completed) none of these work.
Configuration:
Exported monitor JSON - I have set up trigger to true for debug.
{
"name": "a3-fail",
"type": "monitor",
"monitor_type": "bucket_level_monitor",
"enabled": true,
"schedule": {
"period": {
"unit": "MINUTES",
"interval": 1
}
},
"inputs": [
{
"search": {
"indices": [
"hercules-http*"
],
"query": {
"size": 0,
"query": {
"bool": {
"must": [
{
"term": {
"fields.metadata.adapter.name.keyword": {
"value": "SesameAdapter",
"boost": 1
}
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
},
"aggregations": {
"company_execution": {
"composite": {
"size": 100,
"sources": [
{
"companyId": {
"terms": {
"field": "fields.metadata.client.company.keyword",
"missing_bucket": false,
"order": "asc"
}
}
},
{
"subsidiaryId": {
"terms": {
"field": "fields.metadata.client.subsidiary.keyword",
"missing_bucket": false,
"order": "asc"
}
}
},
{
"executionId": {
"terms": {
"field": "fields.metadata.execution.executionId.keyword",
"missing_bucket": false,
"order": "asc"
}
}
}
]
},
"aggregations": {
"status_401": {
"filter": {
"term": {
"fields.metadata.response.status": {
"value": 400,
"boost": 1
}
}
}
}
}
}
}
}
}
}
],
"triggers": [
{
"bucket_level_trigger": {
"id": "i4BB2ZoBr7VeUFolZsXg",
"name": "test",
"severity": "1",
"condition": {
"buckets_path": {
"count_401": "status_401._count"
},
"parent_bucket_path": "company_execution",
"script": {
"source": "true",
"lang": "painless"
},
"gap_policy": "skip"
},
"actions": [
{
"id": "notification521895",
"name": "test",
"destination_id": "aE4hVJoBr7VeUFolOKXJ",
"message_template": {
"source": "Monitor {{ctx.monitor.name}} just entered alert status. Please investigate the issue.\n - Trigger: {{ctx.trigger.name}}\n - Severity: {{ctx.trigger.severity}}\n - Period start: {{ctx.periodStart}} UTC\n - Period end: {{ctx.periodEnd}} UTC\n\n - Deduped Alerts:\n {{#ctx.dedupedAlerts}}\n * {{id}} : {{bucket_keys}}\n {{/ctx.dedupedAlerts}}\n\n - New Alerts:\n {{#ctx.newAlerts}}\n * {{id}} : {{bucket_keys}}\n {{/ctx.newAlerts}}\n\n - Completed Alerts:\n {{#ctx.completedAlerts}}\n * {{id}} : {{bucket_keys}}\n {{/ctx.completedAlerts}}",
"lang": "mustache"
},
"throttle_enabled": false,
"subject_template": {
"source": "Alerting Notification action",
"lang": "mustache"
},
"action_execution_policy": {
"action_execution_scope": {
"per_execution": {}
}
}
}
]
}
}
],
"ui_metadata": {
"schedule": {
"timezone": null,
"frequency": "interval",
"period": {
"unit": "MINUTES",
"interval": 1
},
"daily": 0,
"weekly": {
"tue": false,
"wed": false,
"thur": false,
"sat": false,
"fri": false,
"mon": false,
"sun": false
},
"monthly": {
"type": "day",
"day": 1
},
"cronExpression": "0 */1 * * *"
},
"monitor_type": "bucket_level_monitor",
"search": {
"searchType": "query",
"timeField": "@timestamp",
"aggregations": [],
"groupBy": [],
"bucketValue": 1,
"bucketUnitOfTime": "h",
"filters": []
}
}
}
Manual execution of the monitor
{
"monitor_name": "a3-fail",
"period_start": 1764589179460,
"period_end": 1764589239460,
"error": null,
"input_results": {
"results": [
{
"_shards": {
"total": 10,
"failed": 0,
"successful": 10,
"skipped": 0
},
"hits": {
"hits": [],
"total": {
"value": 3,
"relation": "eq"
},
"max_score": null
},
"took": 3,
"timed_out": false,
"aggregations": {
"company_execution": {
"buckets": [
{
"doc_count": 1,
"status_401": {
"doc_count": 0
},
"key": {
"executionId": "0f4d94d5-fe54-4868-8e08-495cf595beb9",
"companyId": "[REDACTED]",
"subsidiaryId": "[REDACTED]"
}
},
{
"doc_count": 1,
"status_401": {
"doc_count": 0
},
"key": {
"executionId": "9450701a-7488-4266-9404-7ba0db218594",
"companyId": "[REDACTED]",
"subsidiaryId": "[REDACTED]"
}
},
{
"doc_count": 1,
"status_401": {
"doc_count": 0
},
"key": {
"executionId": "e1b64099-5acd-44f2-865b-157b55516fb6",
"companyId": "[REDACTED]",
"subsidiaryId": "[REDACTED]"
}
}
],
"after_key": {
"executionId": "e1b64099-5acd-44f2-865b-157b55516fb6",
"companyId": "[REDACTED]",
"subsidiaryId": "[REDACTED]"
}
},
"i4BB2ZoBr7VeUFolZsXg": {
"parent_bucket_path": "company_execution",
"bucket_indices": [
0,
1,
2
]
}
}
}
],
"error": null
},
"trigger_results": {
"i4BB2ZoBr7VeUFolZsXg": {
"name": "test",
"agg_result_buckets": {
"0f4d94d5-fe54-4868-8e08-495cf595beb9#B19325786#B19325786": {
"agg_alert_content": {
"parent_bucket_path": "company_execution",
"bucket_keys": [
"0f4d94d5-fe54-4868-8e08-495cf595beb9",
"[REDACTED]",
"[REDACTED]"
],
"bucket": {
"doc_count": 1,
"status_401": {
"doc_count": 0
},
"key": {
"executionId": "0f4d94d5-fe54-4868-8e08-495cf595beb9",
"companyId": "[REDACTED]",
"subsidiaryId": "[REDACTED]"
}
}
}
},
"9450701a-7488-4266-9404-7ba0db218594#B19325786#B19325786": {
"agg_alert_content": {
"parent_bucket_path": "company_execution",
"bucket_keys": [
"9450701a-7488-4266-9404-7ba0db218594",
"[REDACTED]",
"[REDACTED]"
],
"bucket": {
"doc_count": 1,
"status_401": {
"doc_count": 0
},
"key": {
"executionId": "9450701a-7488-4266-9404-7ba0db218594",
"companyId": "[REDACTED]",
"subsidiaryId": "[REDACTED]"
}
}
}
},
"e1b64099-5acd-44f2-865b-157b55516fb6#B19325786#B19325786": {
"agg_alert_content": {
"parent_bucket_path": "company_execution",
"bucket_keys": [
"e1b64099-5acd-44f2-865b-157b55516fb6",
"[REDACTED]",
"[REDACTED]"
],
"bucket": {
"doc_count": 1,
"status_401": {
"doc_count": 0
},
"key": {
"executionId": "e1b64099-5acd-44f2-865b-157b55516fb6",
"companyId": "[REDACTED]",
"subsidiaryId": "[REDACTED]"
}
}
}
}
},
"action_results": {},
"error": null
}
}
}
Execution with dryrun
{
"monitor_name": "a3-fail",
"period_start": 1764589267681,
"period_end": 1764589327681,
"error": null,
"input_results": {
"results": [
{
"_shards": {
"total": 10,
"failed": 0,
"successful": 10,
"skipped": 0
},
"hits": {
"hits": [],
"total": {
"value": 3,
"relation": "eq"
},
"max_score": null
},
"took": 2,
"timed_out": false,
"aggregations": {
"company_execution": {
"buckets": [
{
"doc_count": 1,
"status_401": {
"doc_count": 0
},
"key": {
"executionId": "0f4d94d5-fe54-4868-8e08-495cf595beb9",
"companyId": "[REDACTED]",
"subsidiaryId": "[REDACTED]"
}
},
{
"doc_count": 1,
"status_401": {
"doc_count": 0
},
"key": {
"executionId": "9450701a-7488-4266-9404-7ba0db218594",
"companyId": "[REDACTED]",
"subsidiaryId": "[REDACTED]"
}
},
{
"doc_count": 1,
"status_401": {
"doc_count": 0
},
"key": {
"executionId": "e1b64099-5acd-44f2-865b-157b55516fb6",
"companyId": "[REDACTED]",
"subsidiaryId": "[REDACTED]"
}
}
],
"after_key": {
"executionId": "e1b64099-5acd-44f2-865b-157b55516fb6",
"companyId": "[REDACTED]",
"subsidiaryId": "[REDACTED]"
}
},
"i4BB2ZoBr7VeUFolZsXg": {
"parent_bucket_path": "company_execution",
"bucket_indices": [
0,
1,
2
]
}
}
}
],
"error": null
},
"trigger_results": {
"i4BB2ZoBr7VeUFolZsXg": {
"name": "test",
"agg_result_buckets": {
"0f4d94d5-fe54-4868-8e08-495cf595beb9#B19325786#B19325786": {
"agg_alert_content": {
"parent_bucket_path": "company_execution",
"bucket_keys": [
"0f4d94d5-fe54-4868-8e08-495cf595beb9",
"[REDACTED]",
"[REDACTED]"
],
"bucket": {
"doc_count": 1,
"status_401": {
"doc_count": 0
},
"key": {
"executionId": "0f4d94d5-fe54-4868-8e08-495cf595beb9",
"companyId": "[REDACTED]",
"subsidiaryId": "[REDACTED]"
}
}
}
},
"9450701a-7488-4266-9404-7ba0db218594#B19325786#B19325786": {
"agg_alert_content": {
"parent_bucket_path": "company_execution",
"bucket_keys": [
"9450701a-7488-4266-9404-7ba0db218594",
"[REDACTED]",
"[REDACTED]"
],
"bucket": {
"doc_count": 1,
"status_401": {
"doc_count": 0
},
"key": {
"executionId": "9450701a-7488-4266-9404-7ba0db218594",
"companyId": "[REDACTED]",
"subsidiaryId": "[REDACTED]"
}
}
}
},
"e1b64099-5acd-44f2-865b-157b55516fb6#B19325786#B19325786": {
"agg_alert_content": {
"parent_bucket_path": "company_execution",
"bucket_keys": [
"e1b64099-5acd-44f2-865b-157b55516fb6",
"[REDACTED]",
"[REDACTED]"
],
"bucket": {
"doc_count": 1,
"status_401": {
"doc_count": 0
},
"key": {
"executionId": "e1b64099-5acd-44f2-865b-157b55516fb6",
"companyId": "[REDACTED]",
"subsidiaryId": "[REDACTED]"
}
}
}
}
},
"action_results": {
"0f4d94d5-fe54-4868-8e08-495cf595beb9#B19325786#B19325786": {
"notification521895": {
"id": "notification521895",
"name": "test",
"output": {
"subject": "Alerting Notification action",
"message": """Monitor a3-fail just entered alert status. Please investigate the issue.
- Trigger: test
- Severity: 1
- Period start: 2025-12-01T11:41:07.681Z UTC
- Period end: 2025-12-01T11:42:07.681Z UTC
- Deduped Alerts:
- New Alerts:
* : 0f4d94d5-fe54-4868-8e08-495cf595beb9,[REDACTED],[REDACTED]
* : 9450701a-7488-4266-9404-7ba0db218594,[REDACTED],[REDACTED]
* : e1b64099-5acd-44f2-865b-157b55516fb6,[REDACTED],[REDACTED]
- Completed Alerts:
"""
},
"throttled": false,
"executionTime": 1764589327531,
"error": null
}
},
"9450701a-7488-4266-9404-7ba0db218594#B19325786#B19325786": {
"notification521895": {
"id": "notification521895",
"name": "test",
"output": {
"subject": "Alerting Notification action",
"message": """Monitor a3-fail just entered alert status. Please investigate the issue.
- Trigger: test
- Severity: 1
- Period start: 2025-12-01T11:41:07.681Z UTC
- Period end: 2025-12-01T11:42:07.681Z UTC
- Deduped Alerts:
- New Alerts:
* : 0f4d94d5-fe54-4868-8e08-495cf595beb9,[REDACTED],[REDACTED]
* : 9450701a-7488-4266-9404-7ba0db218594,[REDACTED],[REDACTED]
* : e1b64099-5acd-44f2-865b-157b55516fb6,[REDACTED],[REDACTED]
- Completed Alerts:
"""
},
"throttled": false,
"executionTime": 1764589327531,
"error": null
}
},
"e1b64099-5acd-44f2-865b-157b55516fb6#B19325786#B19325786": {
"notification521895": {
"id": "notification521895",
"name": "test",
"output": {
"subject": "Alerting Notification action",
"message": """Monitor a3-fail just entered alert status. Please investigate the issue.
- Trigger: test
- Severity: 1
- Period start: 2025-12-01T11:41:07.681Z UTC
- Period end: 2025-12-01T11:42:07.681Z UTC
- Deduped Alerts:
- New Alerts:
* : 0f4d94d5-fe54-4868-8e08-495cf595beb9,[REDACTED],[REDACTED]
* : 9450701a-7488-4266-9404-7ba0db218594,[REDACTED],[REDACTED]
* : e1b64099-5acd-44f2-865b-157b55516fb6,[REDACTED],[REDACTED]
- Completed Alerts:
"""
},
"throttled": false,
"executionTime": 1764589327531,
"error": null
}
}
},
"error": null
}
}
}
Relevant Logs or Screenshots:
Extra information:
I have tried the same query on a simple monitor, works fine, gets executed but i need bucket level monitor. On the other hand, I have tried this query on a different environment (staging) and works.
I’m using aws managed OpenSearch.