Hello,
I am seeking your explanations and help with my issue.
I am using Opendistro alerting plugin with an ELK stack 7.6.1 for security monitoring.
Lately I noticed that some monitors are triggered but now alerts were sent to my destination (TheHive). When I manually do the tests again with the POST _opendistro/_alerting/monitors/<MonitorID>/_execute I receive the alert on TheHive with no errors like this :
But when I leave it as it is I notice that the monitor is triggered but nothing is sent to my destination :
Question °1 : How can I start troubleshooting those 6 errors showing above since when i _execute it manually no errors are detected ?
I am not acknowledging any alerts so that I can keep receiving every alert triggered and what I’ve noticed is that this behavior is followed by the alert being in DELETE state then after next trigger that results to false the sate becomes COMPLETED:
Question °2: Why the state of the alert became DELETE ? in the documentation DELETED means “Someone deleted the monitor or trigger associated with this alert while the alert was ongoing.” I don’t see why this should be the case in my monitors.
Question °3 : Can having multiple monitors, with the same Trigger name where each one have different loginc from monitor to another introduce this kind of behavior
I configured my monitors directly with kibana’s Dev Tools using the following API calls:
Create : POST _opendistro/_alerting/monitors
Update : PUT _opendistro/_alerting/monitors/<MonitorID>
Monitor Sample (REDACTED Version) :
{
"type": "monitor",
"name": "PV-COM-01- Detect FTP (File Transfer Protocol) Activity to the Internet",
"enabled": true,
"schedule": {
"period": {
"interval": 5,
"unit": "MINUTES"
}
},
"inputs": [
{
"search": {
"indices": [
"REDACTED*"
],
"query": {
"aggregations": {},
"size": 1,
"query": {
"bool": {
"must": [
{
"match": {
"tags": "firewall"
}
},
{
"term": {
"source.locality": {
"value": "private"
}
}
},
{
"term": {
"destination.locality": {
"value": "public"
}
}
},
{
"terms": {
"destination.port": [
"20",
"21"
]
}
},
{
"range": {
"@timestamp": {
"from": "{{period_end}}||-5m",
"to": "{{period_end}}"
}
}
}
]
}
},
"aggs": {
"by_src_ip": {
"terms": {
"field": "source.ip"
},
"aggs": {
"by_dst_port": {
"terms": {
"field": "destination.port"
}
}
}
}
},
"sort": [
{
"@timestamp": {
"order": "asc"
}
}
]
}
}
}
],
"triggers": [
{
"name": "PV-COM-01",
"severity": "2",
"condition": {
"script": {
"source": """boolean incident = false;
for (int i = 0; i < ctx.results[0].aggregations.by_src_ip.buckets.size(); i++) {
for (int j = 0; j < ctx.results[0].aggregations.by_src_ip.buckets[i].by_dst_port.buckets.size(); j++) {
if (ctx.results[0].aggregations.by_src_ip.buckets[i].by_dst_port.buckets[j].doc_count > 0) {
incident = true;
}
}
}
return incident;""",
"lang": "painless"
}
},
"actions": [
{
"name": "thehive",
"destination_id": "Q-4vmXUBWxGnc2wbFu2N",
"message_template": {
"source": "REDACTED"
},
"throttle_enabled": false
}
]
}
]
}
Troubleshooting :
-
Input:
GET _cat/thread_pool/opendistro_monitor_runner?v&h=id,name,active,error,completed -
output : nothing
-
Input :
GET _opendistro/_alerting/stats -
Output :
{
"_nodes" : {
"total" : 3,
"successful" : 3,
"failed" : 0
},
"cluster_name" : "REDACTED",
"opendistro.scheduled_jobs.enabled" : true,
"scheduled_job_index_exists" : true,
"scheduled_job_index_status" : "green",
"nodes_on_schedule" : 3,
"nodes_not_on_schedule" : 0,
"nodes" : {
"z6aTr_knROygmLRXAzauGQ" : {
"name" : "kibanamaster",
"schedule_status" : "green",
"roles" : [
"MASTER"
],
"job_scheduling_metrics" : {
"last_full_sweep_time_millis" : 219239,
"full_sweep_on_time" : true
},
"jobs_info" : { }
},
"defcJ92eSZ6K8aF1-VlrGQ" : {
"name" : "masterdatahot",
"schedule_status" : "green",
"roles" : [
"INGEST",
"MASTER",
"DATA"
],
"job_scheduling_metrics" : {
"last_full_sweep_time_millis" : 190648,
"full_sweep_on_time" : true
},
"jobs_info" : {
"vTprzXUBrQ8tcQYYG8tA" : {
"last_execution_time" : 1606489150010,
"running_on_time" : true
},
"MMyLwXUBrQ8tcQYYOtkv" : {
"last_execution_time" : 1606489128486,
"running_on_time" : true
},
"q3-RzXUBrQ8tcQYYSoEA" : {
"last_execution_time" : 1606488952344,
"running_on_time" : true
},
"xsBfqXUBWxGnc2wb59cC" : {
"last_execution_time" : 1606489115963,
"running_on_time" : true
},
"rZ06qHUBWxGnc2wben5v" : {
"last_execution_time" : 1606488906119,
"running_on_time" : true
},
"SVFxvHUBrQ8tcQYYvizi" : {
"last_execution_time" : 1606488772442,
"running_on_time" : true
},
"61wpqXUBWxGnc2wb58f1" : {
"last_execution_time" : 1606488997261,
"running_on_time" : true
},
"XCZJmXUBWxGnc2wb2Q3X" : {
"last_execution_time" : 1606489115346,
"running_on_time" : true
},
"_EdyzXUBrQ8tcQYYTLLb" : {
"last_execution_time" : 1606489021416,
"running_on_time" : true
},
"Q5hKqXUBWxGnc2wbHmc9" : {
"last_execution_time" : 1606489008310,
"running_on_time" : true
},
"zhAuwnUBrQ8tcQYYqMBs" : {
"last_execution_time" : 1606489039077,
"running_on_time" : true
},
"sZJjBnYB_pltEOSpmuC0" : {
"last_execution_time" : 1606486159660,
"running_on_time" : true
},
"u4tDqXUBWxGnc2wbIlUW" : {
"last_execution_time" : 1606489150544,
"running_on_time" : true
},
"t0RwzXUBrQ8tcQYYkMuq" : {
"last_execution_time" : 1606488907747,
"running_on_time" : true
},
"l-jLzXUBrQ8tcQYYcUTO" : {
"last_execution_time" : 1606489163623,
"running_on_time" : true
},
"EEVrvHUBrQ8tcQYYztww" : {
"last_execution_time" : 1606489103143,
"running_on_time" : true
},
"y_t-qXUBWxGnc2wb5xm6" : {
"last_execution_time" : 1606489107762,
"running_on_time" : true
}
}
},
"vn5D0097SFyZrwjp59Lg0g" : {
"name" : "masterdatawarm",
"schedule_status" : "green",
"roles" : [
"MASTER",
"DATA"
],
"job_scheduling_metrics" : {
"last_full_sweep_time_millis" : 195437,
"full_sweep_on_time" : true
},
"jobs_info" : {
"LkFpvHUBrQ8tcQYYbU74" : {
"last_execution_time" : 1606489187440,
"running_on_time" : true
},
"alQGznUBrQ8tcQYY-y8E" : {
"last_execution_time" : 1606489165370,
"running_on_time" : true
},
"nRSMqXUBWxGnc2wbO5Vp" : {
"last_execution_time" : 1606489141154,
"running_on_time" : true
},
"aFYmqXUBWxGnc2wbn3q-" : {
"last_execution_time" : 1606489082169,
"running_on_time" : true
},
"kkpzzXUBrQ8tcQYY65W2" : {
"last_execution_time" : 1606489127660,
"running_on_time" : true
},
"ffV7qXUBWxGnc2wbti-Z" : {
"last_execution_time" : 1606489138577,
"running_on_time" : true
},
"NYQ_qXUBWxGnc2wbf8wM" : {
"last_execution_time" : 1606488912197,
"running_on_time" : true
},
"nLSlvHUBrQ8tcQYYcPuM" : {
"last_execution_time" : 1606489160259,
"running_on_time" : true
},
"sQhLvHUBrQ8tcQYY1m2f" : {
"last_execution_time" : 1606488688138,
"running_on_time" : true
},
"DfxtqHUBWxGnc2wb8Pco" : {
"last_execution_time" : 1606488978595,
"running_on_time" : true
},
"fHs6qXUBWxGnc2wbQx-I" : {
"last_execution_time" : 1606489169281,
"running_on_time" : true
},
"WDlqzXUBrQ8tcQYYICvp" : {
"last_execution_time" : 1606489085923,
"running_on_time" : true
},
"JzJmzXUBrQ8tcQYYeE72" : {
"last_execution_time" : 1606489146307,
"running_on_time" : true
},
"DXFNBnYB_pltEOSpRj4o" : {
"last_execution_time" : 1606488296217,
"running_on_time" : true
},
"cXU2qXUBWxGnc2wb-jQI" : {
"last_execution_time" : 1606488953858,
"running_on_time" : true
},
"GD1szXUBrQ8tcQYYekuV" : {
"last_execution_time" : 1606488939949,
"running_on_time" : true
},
"STVozXUBrQ8tcQYYEIcg" : {
"last_execution_time" : 1606488950552,
"running_on_time" : true
},
"jT9tzXUBrQ8tcQYY4ax6" : {
"last_execution_time" : 1606489031825,
"running_on_time" : true
},
"zmOBzXUBrQ8tcQYY-Jgm" : {
"last_execution_time" : 1606489148350,
"running_on_time" : true
},
"PIJkwXUBrQ8tcQYY9z5g" : {
"last_execution_time" : 1606489021018,
"running_on_time" : true
},
"N5ugzXUBrQ8tcQYYtVeT" : {
"last_execution_time" : 1606489062922,
"running_on_time" : true
},
"hryzzXUBrQ8tcQYYEE-G" : {
"last_execution_time" : 1606489065854,
"running_on_time" : true
},
"s-d0qXUBWxGnc2wb3-mJ" : {
"last_execution_time" : 1606489170307,
"running_on_time" : true
},
"sUFvzXUBrQ8tcQYYAbgu" : {
"last_execution_time" : 1606489105476,
"running_on_time" : true
},
"vS1jzXUBrQ8tcQYY_vxx" : {
"last_execution_time" : 1606488983836,
"running_on_time" : true
}
}
}
}
}
- Input :
GET /.opendistro-alerting-alerts/_search
{
"query": {
"terms": {
"_routing": [
"false"
]
}
}
}
- Output :
{
"took" : 2,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 0,
"relation" : "eq"
},
"max_score" : null,
"hits" : [ ]
}
}
Thank you for your help
