Hi everyone,
==============================================================================
solution just HERE , for who want to use Opsgenie:
==============================================================================
i need some tips for OpsGenie integration. I’m on:
opendistro_alerting-0.9.0.0 plugin installed on ES cluster and Kibana.
elasticsearch 6.7.1 cluster based on elasticsearch official Docker repository
kibana 6.7.1 based on elasticsearch official Docker repository
On ES cluster nodes:
sh-4.2# /usr/share/elasticsearch/bin/elasticsearch-plugin list
opendistro_alerting
On kibana node:
h-4.2$ /usr/share/kibana/bin/kibana-plugin list
opendistro-alerting@0.9.0.0
Opsgenie API :
Exemple with Curl to creat ean alert (tested on my kibana node, it’s work well) :
curl -X POST https://api.eu.opsgenie.com/v2/alerts \
-H "Content-Type: application/json" \
-H "Authorization: GenieKey xxx-xxxx-xxxx-xxxx-xxxxxxxxxx" \
-d '{ "message": "An example alert message" }'
I have on monitor, who trigger 1 custom webhook, for testing.
My custom webhook on alerting module:
My action on the monitor’s trigger:
I check on Kibana and opsgenie, no errors or everything else when i use the “Test message” function.
Kibana logs when i click on “send test message” :
{“type”:“response”,“@timestamp”:“2019-06-18T08:05:37Z”,“tags”:,“pid”:1,“method”:“post”,“statusCode”:200,“req”:{“url”:“/api/alerting/monitors/_execute?dryrun=false”,“method”:“post”,“headers”:{“content-length”:“1598”,“accept”:“application/json, text/plain, /”,“origin”:“http://kibana-opendistro-plugin-graylog.apps.xxx.xxx”,“kbn-version”:“6.7.1”,“user-agent”:“Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36”,“content-type”:“application/json;charset=UTF-8”,“referer”:“http://kibana-opendistro-plugin-graylog.apps.xxx.xxx/s/xxx-monitoring/app/opendistro-alerting",“accept-encoding”:"gzip , deflate”,“accept-language”:“fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7,it;q=0.6,de;q=0.5”,“host”:“kibana-opendistro-plugin-graylog.apps.xxx.xxx”,“x-forwarded-host”:“kibana-opendistro-plugin-graylog.apps.xxx.xxx”,“x-forwarded-port”:“80”,“x-forwarded-proto”:“http”,“forwarded”:“for=10.84.70.240;host=kibana-opendistro-plugin-graylog.apps.xxx.xxx;proto=http”,“x-forwarded-for”:“10.84.70.240”},“remoteAddress”:“10.131.4.1”,“userAgent”:“10.131.4.1”,“referer”:“http://kibana-opendistro-plugin-graylog.apps.xxx.xxx/s/xxx-monitoring/app/opendistro-alerting"},“res”:{“statusCode”:200,“responseTime”:277,“contentLength”:9},“message”:"POST /api/alerting/monitors/_execute?dryrun=false 200 277ms - 9.0B”}
{“type”:“response”,“@timestamp”:“2019-06-18T08:05:38Z”,“tags”:,“pid”:1,“method”:“post”,“statusCode”:200,“req”:{“url”:“/api/alerting/monitors/_execute?dryrun=false”,“method”:“post”,“headers”:{“content-length”:“1598”,“accept”:“application/json, text/plain, /”,“origin”:“http://kibana-opendistro-plugin-graylog.apps.xxx.xxx”,“kbn-version”:“6.7.1”,“user-agent”:“Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36”,“content-type”:“application/json;charset=UTF-8”,“referer”:“http://kibana-opendistro-plugin-graylog.apps.xxx.xxx/s/xxx-monitoring/app/opendistro-alerting",“accept-encoding”:"gzip , deflate”,“accept-language”:“fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7,it;q=0.6,de;q=0.5”,“host”:“kibana-opendistro-plugin-graylog.apps.xxx.xxx”,“x-forwarded-host”:“kibana-opendistro-plugin-graylog.apps.xxx.xxx”,“x-forwarded-port”:“80”,“x-forwarded-proto”:“http”,“forwarded”:“for=10.84.70.240;host=kibana-opendistro-plugin-graylog.apps.xxx.xxx;proto=http”,“x-forwarded-for”:“10.84.70.240”},“remoteAddress”:“10.131.4.1”,“userAgent”:“10.131.4.1”,“referer”:“http://kibana-opendistro-plugin-graylog.apps.xxx.xxx/s/xxx-monitoring/app/opendistro-alerting"},“res”:{“statusCode”:200,“responseTime”:34,“contentLength”:9},“message”:"POST /api/alerting/monitors/_execute?dryrun=false 200 34ms - 9.0B”}
.opendistro-alerting-config index looks empty 
{
".opendistro-alerting-config": {
"aliases": {
},
"mappings": {
"_doc": {
"properties": {
"destination": {
"dynamic": "false",
"properties": {
"chime": {
"properties": {
"url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"custom_webhook": {
"properties": {
"header_params": {
"type": "object",
"enabled": false
},
"host": {
"type": "text"
},
"password": {
"type": "text"
},
"path": {
"type": "keyword"
},
"port": {
"type": "integer"
},
"query_params": {
"type": "object",
"enabled": false
},
"scheme": {
"type": "keyword"
},
"url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"username": {
"type": "text"
}
}
},
"last_update_time": {
"type": "date",
"format": "strict_date_time||epoch_millis"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"slack": {
"properties": {
"url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"type": {
"type": "keyword"
}
}
},
"monitor": {
"dynamic": "false",
"properties": {
"enabled": {
"type": "boolean"
},
"enabled_time": {
"type": "date",
"format": "strict_date_time||epoch_millis"
},
"inputs": {
"type": "nested",
"properties": {
"search": {
"properties": {
"indices": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"query": {
"type": "object",
"enabled": false
}
}
}
}
},
"last_update_time": {
"type": "date",
"format": "strict_date_time||epoch_millis"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"schedule": {
"properties": {
"cron": {
"properties": {
"expression": {
"type": "text"
},
"timezone": {
"type": "keyword"
}
}
},
"period": {
"properties": {
"interval": {
"type": "integer"
},
"unit": {
"type": "keyword"
}
}
}
}
},
"triggers": {
"type": "nested",
"properties": {
"actions": {
"type": "nested",
"properties": {
"destination_id": {
"type": "keyword"
},
"message_template": {
"type": "object",
"enabled": false
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"subject_template": {
"type": "object",
"enabled": false
}
}
},
"condition": {
"type": "object",
"enabled": false
},
"min_time_between_executions": {
"type": "integer"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"type": {
"type": "keyword"
},
"ui_metadata": {
"type": "object",
"enabled": false
}
}
}
}
}
},
"settings": {
"index": {
"creation_date": "1560425771825",
"number_of_shards": "5",
"number_of_replicas": "1",
"uuid": "CKZ-5k4CSEiqhjyVAzHpuw",
"version": {
"created": "6070199"
},
"provided_name": ".opendistro-alerting-config"
}
}
}
}
Logs on cluster side :
[2019-06-18T09:21:04,490][INFO ][c.a.o.a.d.c.DestinationHttpClient] [es-cluster-2] endpoint empty. Fall back to host:port/path
[2019-06-18T09:21:04,524][ERROR][c.a.o.a.d.f.CustomWebhookDestinationFactory] [es-cluster-2] Exception publishing Message: DestinationType: CUSTOMWEBHOOK, DestinationName:Opsgenie_XXXXX, Url: , scheme: https, Host: api.eu.opsgenie.com, Port: 443, Path: null, Message: Monitor test just entered an alert state. Please investigate the issue.
Send test message
Message preview
- Trigger: test
- Severity: 1
- Period start: 2019-06-18T09:20:04.484Z
- Period end: 2019-06-18T09:21:04.484Z
java.io.IOException: Failed: HttpResponseProxy{HTTP/1.1 404 Not Found [Content-Type: application/json; charset=UTF-8, Transfer-Encoding: chunked, Connection: keep-alive, Date: Tue, 18 Jun 2019 09:21:04 GMT, Allow: HEAD, GET, X-Response-Time: 0.002, X-Request-ID: ad95db49-64ca-461e-af95-9eaee870ae68, X-Cache: Error from cloudfront, Via: 1.1 3ccfbae98f5816b531634c1e82e45259.cloudfront.net (CloudFront), X-Amz-Cf-Pop: FRA50, X-Amz-Cf-Id: j5a5w82Z-DyyZDFdrfXQIFaou3t3TYTQ5dj3i-Cx8SzxvwDiXszNPg==] ResponseEntityProxy{[Content-Type: application/json; charset=UTF-8,Chunked: true]}}
at com.amazon.opendistroforelasticsearch.alerting.destination.client.DestinationHttpClient.validateResponseStatus(DestinationHttpClient.java:156) ~[alerting-notification-0.9.0.0.jar:?]
at com.amazon.opendistroforelasticsearch.alerting.destination.client.DestinationHttpClient.execute(DestinationHttpClient.java:81) ~[alerting-notification-0.9.0.0.jar:?]
at com.amazon.opendistroforelasticsearch.alerting.destination.factory.CustomWebhookDestinationFactory.publish(CustomWebhookDestinationFactory.java:42) [alerting-notification-0.9.0.0.jar:?]
at com.amazon.opendistroforelasticsearch.alerting.destination.factory.CustomWebhookDestinationFactory.publish(CustomWebhookDestinationFactory.java:29) [alerting-notification-0.9.0.0.jar:?]
at com.amazon.opendistroforelasticsearch.alerting.destination.Notification.lambda$publish$0(Notification.java:43) [alerting-notification-0.9.0.0.jar:?]
at java.security.AccessController.doPrivileged(AccessController.java:310) [?:?]
at com.amazon.opendistroforelasticsearch.alerting.destination.Notification.publish(Notification.java:41) [alerting-notification-0.9.0.0.jar:?]
at com.amazon.opendistroforelasticsearch.alerting.model.destination.Destination.publish(Destination.kt:167) [opendistro_alerting-0.9.0.0.jar:0.9.0.0]
at com.amazon.opendistroforelasticsearch.alerting.MonitorRunner.runAction(MonitorRunner.kt:393) [opendistro_alerting-0.9.0.0.jar:0.9.0.0]
at com.amazon.opendistroforelasticsearch.alerting.MonitorRunner.runMonitor(MonitorRunner.kt:196) [opendistro_alerting-0.9.0.0.jar:0.9.0.0]
at com.amazon.opendistroforelasticsearch.alerting.resthandler.RestExecuteMonitorAction$prepareRequest$1$executeMonitor$1$1.run(RestExecuteMonitorAction.kt:65) [opendistro_alerting-0.9.0.0.jar:0.9.0.0]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) [elasticsearch-6.7.1.jar:6.7.1]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]
at java.lang.Thread.run(Thread.java:835) [?:?]
[2019-06-18T09:21:04,525][INFO ][c.a.o.a.m.MonitorRunResult] [es-cluster-2] Internal error: java.io.IOException: Failed: HttpResponseProxy{HTTP/1.1 404 Not Found [Content-Type: application/json; charset=UTF-8, Transfer-Encoding: chunked, Connection: keep-alive, Date: Tue, 18 Jun 2019 09:21:04 GMT, Allow: HEAD, GET, X-Response-Time: 0.002, X-Request-ID: ad95db49-64ca-461e-af95-9eaee870ae68, X-Cache: Error from cloudfront, Via: 1.1 3ccfbae98f5816b531634c1e82e45259.cloudfront.net (CloudFront), X-Amz-Cf-Pop: FRA50, X-Amz-Cf-Id: j5a5w82Z-DyyZDFdrfXQIFaou3t3TYTQ5dj3i-Cx8SzxvwDiXszNPg==] ResponseEntityProxy{[Content-Type: application/json; charset=UTF-8,Chunked: true]}}. See the Elasticsearch.log for details
java.lang.IllegalStateException: java.io.IOException: Failed: HttpResponseProxy{HTTP/1.1 404 Not Found [Content-Type: application/json; charset=UTF-8, Transfer-Encoding: chunked, Connection: keep-alive, Date: Tue, 18 Jun 2019 09:21:04 GMT, Allow: HEAD, GET, X-Response-Time: 0.002, X-Request-ID: ad95db49-64ca-461e-af95-9eaee870ae68, X-Cache: Error from cloudfront, Via: 1.1 3ccfbae98f5816b531634c1e82e45259.cloudfront.net (CloudFront), X-Amz-Cf-Pop: FRA50, X-Amz-Cf-Id: j5a5w82Z-DyyZDFdrfXQIFaou3t3TYTQ5dj3i-Cx8SzxvwDiXszNPg==] ResponseEntityProxy{[Content-Type: application/json; charset=UTF-8,Chunked: true]}}
at com.amazon.opendistroforelasticsearch.alerting.destination.factory.CustomWebhookDestinationFactory.publish(CustomWebhookDestinationFactory.java:46) ~[alerting-notification-0.9.0.0.jar:?]
at com.amazon.opendistroforelasticsearch.alerting.destination.factory.CustomWebhookDestinationFactory.publish(CustomWebhookDestinationFactory.java:29) ~[alerting-notification-0.9.0.0.jar:?]
at com.amazon.opendistroforelasticsearch.alerting.destination.Notification.lambda$publish$0(Notification.java:43) ~[alerting-notification-0.9.0.0.jar:?]
at java.security.AccessController.doPrivileged(AccessController.java:310) ~[?:?]
at com.amazon.opendistroforelasticsearch.alerting.destination.Notification.publish(Notification.java:41) ~[alerting-notification-0.9.0.0.jar:?]
at com.amazon.opendistroforelasticsearch.alerting.model.destination.Destination.publish(Destination.kt:167) ~[opendistro_alerting-0.9.0.0.jar:0.9.0.0]
at com.amazon.opendistroforelasticsearch.alerting.MonitorRunner.runAction(MonitorRunner.kt:393) ~[opendistro_alerting-0.9.0.0.jar:0.9.0.0]
at com.amazon.opendistroforelasticsearch.alerting.MonitorRunner.runMonitor(MonitorRunner.kt:196) [opendistro_alerting-0.9.0.0.jar:0.9.0.0]
at com.amazon.opendistroforelasticsearch.alerting.resthandler.RestExecuteMonitorAction$prepareRequest$1$executeMonitor$1$1.run(RestExecuteMonitorAction.kt:65) [opendistro_alerting-0.9.0.0.jar:0.9.0.0]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) [elasticsearch-6.7.1.jar:6.7.1]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]
at java.lang.Thread.run(Thread.java:835) [?:?]
Caused by: java.io.IOException: Failed: HttpResponseProxy{HTTP/1.1 404 Not Found [Content-Type: application/json; charset=UTF-8, Transfer-Encoding: chunked, Connection: keep-alive, Date: Tue, 18 Jun 2019 09:21:04 GMT, Allow: HEAD, GET, X-Response-Time: 0.002, X-Request-ID: ad95db49-64ca-461e-af95-9eaee870ae68, X-Cache: Error from cloudfront, Via: 1.1 3ccfbae98f5816b531634c1e82e45259.cloudfront.net (CloudFront), X-Amz-Cf-Pop: FRA50, X-Amz-Cf-Id: j5a5w82Z-DyyZDFdrfXQIFaou3t3TYTQ5dj3i-Cx8SzxvwDiXszNPg==] ResponseEntityProxy{[Content-Type: application/json; charset=UTF-8,Chunked: true]}}
at com.amazon.opendistroforelasticsearch.alerting.destination.client.DestinationHttpClient.validateResponseStatus(DestinationHttpClient.java:156) ~[alerting-notification-0.9.0.0.jar:?]
at com.amazon.opendistroforelasticsearch.alerting.destination.client.DestinationHttpClient.execute(DestinationHttpClient.java:81) ~[alerting-notification-0.9.0.0.jar:?]
at com.amazon.opendistroforelasticsearch.alerting.destination.factory.CustomWebhookDestinationFactory.publish(CustomWebhookDestinationFactory.java:42) ~[alerting-notification-0.9.0.0.jar:?]
... 14 more
Any tips to debug it ? 
