I am trying to push docker logs to Elasticsearch using logstash and logspout. But logs are not being pushed to Elasticsearch.
my docker compose file
version: '3'
services:
odfe-node1:
build:
dockerfile: Dockerfile
context: ./elasticsearch
container_name: odfe-node1
environment:
- cluster.name=odfe-cluster
- bootstrap.memory_lock=true # along with the memlock settings below, disables swapping
- "ES_JAVA_OPTS=-Xms512m -Xmx512m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
- opendistro_security.ssl.http.enabled=false
- discovery.zen.ping.unicast.hosts=odfe-node1
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- odfe-data1:/usr/share/elasticsearch/data
ports:
- 9200:9200
#- 9700:9600 # required for Performance Analyzer
networks:
- odfe-net
odfe-node2:
image: amazon/opendistro-for-elasticsearch:0.7.0
container_name: odfe-node2
environment:
- cluster.name=odfe-cluster
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- discovery.zen.ping.unicast.hosts=odfe-node1
- opendistro_security.ssl.http.enabled=false
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- odfe-data2:/usr/share/elasticsearch/data
- ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
networks:
- odfe-net
kibana:
build:
dockerfile: Dockerfile
context: ./kibana
container_name: odfe-kibana
ports:
- 5601:5601
expose:
- "5601"
environment:
ELASTICSEARCH_URL: http://odfe-node1:9200
OPENDISTRO_SECURITY_MULTITENANCY_ENABLED: "false"
XPACK_SECURITY_ENABLED: "false"
networks:
- odfe-net
logstash:
build:
dockerfile: Dockerfile
context: ./logstash
volumes:
- ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro
- ./logstash/pipeline:/usr/share/logstash/pipeline:ro
ports:
- "5000:5000"
- "9600:9600"
environment:
LS_JAVA_OPTS: "-Xmx256m -Xms256m"
networks:
- odfe-net
depends_on:
- odfe-node1
logspout:
image: gliderlabs/logspout:v2
restart: always
command: 'udp://logstash:5044'
#environment:
#ROUTE_URIS: udp://logstash:5000
links:
- logstash
volumes:
- '/var/run/docker.sock:/tmp/docker.sock'
depends_on:
- odfe-node2
- logstash
- kibana
networks:
- odfe-net
volumes:
odfe-data1:
odfe-data2:
networks:
odfe-net:
driver: bridge
Logstash confiugration
input{
udp{
port => 5044
}
}
filter {
if [docker][image] =~ /^logstash/ {
drop { }
}
}
## Add your filters / logstash plugins configuration here
output {
elasticsearch {
hosts => "odfe-node2:9200"
Logstash and Logspout were starting successfully,please refer below logs,
[2019-03-26T08:10:29,250][INFO ][logstash.setting.writabledirectory] Creating directory {:setting=>"path.queue", :path=>"/usr/share/logstash/data/queue"}
[2019-03-26T08:10:29,393][INFO ][logstash.setting.writabledirectory] Creating directory {:setting=>"path.dead_letter_queue", :path=>"/usr/share/logstash/data/dead_letter_queue"}
[2019-03-26T08:10:31,047][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
[2019-03-26T08:10:31,232][INFO ][logstash.runner ] Starting Logstash {"logstash.version"=>"6.6.1"}
[2019-03-26T08:10:31,392][INFO ][logstash.agent ] No persistent UUID file found. Generating new UUID {:uuid=>"dc72a279-239d-4edd-ad44-41b5da944490", :path=>"/usr/share/logstash/data/uuid"}
[2019-03-26T08:11:03,029][INFO ][logstash.pipeline ] Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>2, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50}
[2019-03-26T08:11:05,513][INFO ][logstash.outputs.elasticsearch] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://odfe-node2:9200/]}}
[2019-03-26T08:11:07,080][WARN ][logstash.outputs.elasticsearch] Restored connection to ES instance {:url=>"http://odfe-node2:9200/"}
[2019-03-26T08:11:07,681][INFO ][logstash.outputs.elasticsearch] ES Output version determined {:es_version=>6}
[2019-03-26T08:11:07,715][WARN ][logstash.outputs.elasticsearch] Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type {:es_version=>6}
[2019-03-26T08:11:07,934][INFO ][logstash.outputs.elasticsearch] New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["//odfe-node2:9200"]}
[2019-03-26T08:11:08,085][INFO ][logstash.outputs.elasticsearch] Using mapping template from {:path=>nil}
[2019-03-26T08:11:08,490][INFO ][logstash.outputs.elasticsearch] Attempting to install template {:manage_template=>{"template"=>"logstash-*", "version"=>60001, "settings"=>{"index.refresh_interval"=>"5s"}, "mappings"=>{"_default_"=>{"dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date"}, "@version"=>{"type"=>"keyword"}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}}
[2019-03-26T08:11:08,650][INFO ][logstash.pipeline ] Pipeline started successfully {:pipeline_id=>"main", :thread=>"#<Thread:0x47ac2e3 run>"}
[2019-03-26T08:11:09,249][INFO ][logstash.agent ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
[2019-03-26T08:11:09,329][INFO ][logstash.inputs.udp ] Starting UDP listener {:address=>"0.0.0.0:5044"}
[2019-03-26T08:11:10,395][INFO ][logstash.inputs.udp ] UDP listener started {:address=>"0.0.0.0:5044", :receive_buffer_bytes=>"106496", :queue_size=>"2000"}
[2019-03-26T08:11:12,170][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
Logspout logs
root@server1:~/opendistro-elk-logspout# docker logs 55ce7424f363
2019/03/26 08:08:16 routing all to udp://logstash:5044
2019/03/26 08:08:16 loading and persisting routes in /mnt/routes
2019/03/26 08:08:16 logspout v2 serving http on :8000