Hi,
I am new to OpenSearch. I already have elastic and kibana up and running… now I want to pull log from api (using http poller) and push the log from logstash to elastic… but having this kind of problem.
logstash | [2021-09-28T02:53:05,233][INFO ][logstash.setting.writabledirectory] Creating directory {:setting=>"path.queue", :path=>"/usr/share/logstash/data/queue"}
logstash | [2021-09-28T02:53:05,245][INFO ][logstash.setting.writabledirectory] Creating directory {:setting=>"path.dead_letter_queue", :path=>"/usr/share/logstash/data/dead_letter_queue"}
logstash | [2021-09-28T02:53:05,590][INFO ][logstash.agent ] No persistent UUID file found. Generating new UUID {:uuid=>"bf4885fc-a55a-4c28-895a-a885cc4db297", :path=>"/usr/share/logstash/data/uuid"}
logstash | [2021-09-28T02:53:06,689][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
logstash | [2021-09-28T02:53:07,097][ERROR][logstash.agent ] Failed to execute action {:action=>LogStash::PipelineAction::Create/pipeline_id:main, :exception=>"LogStash::ConfigurationError", :message=>"Expected one of [ \\t\\r\\n], \"#\", \"if\", [A-Za-z0-9_-], '\"', \"'\", \"}\" at line 30, column 1 (byte 638) after output {\n opensearch {\n hosts => [\"https://odfe-node1:9200\"]\n index => \"logstash-index-test\"\n user => \"admin\"\n password => \"admin\"\n ssl => true\n cacert => \"./logstash_pipeline/root-ca.pem\"\n ilm_enabled => false\n }\n\n", :backtrace=>["/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:32:in `compile_imperative'", "org/logstash/execution/AbstractPipelineExt.java:187:in `initialize'", "org/logstash/execution/JavaBasePipelineExt.java:72:in `initialize'", "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:47:in `initialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline_action/create.rb:50:in `execute'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:384:in `block in converge_state'"]}
logstash | [2021-09-28T02:53:07,194][INFO ][logstash.runner ] Logstash shut down.
logstash | [2021-09-28T02:53:07,204][FATAL][org.logstash.Logstash ] Logstash stopped processing because of an error: (SystemExit) exit
logstash | org.jruby.exceptions.SystemExit: (SystemExit) exit
logstash | at org.jruby.RubyKernel.exit(org/jruby/RubyKernel.java:747) ~[jruby-complete-9.2.19.0.jar:?]
logstash | at org.jruby.RubyKernel.exit(org/jruby/RubyKernel.java:710) ~[jruby-complete-9.2.19.0.jar:?]
logstash | at usr.share.logstash.lib.bootstrap.environment.<main>(/usr/share/logstash/lib/bootstrap/environment.rb:94) ~[?:?]
logstash exited with code 1
my docker compose file
version: '3.7'
services:
odfe-node1:
image: amazon/opendistro-for-elasticsearch:1.13.2
container_name: odfe-node1
environment:
- cluster.name=odfe-cluster
- node.name=odfe-node1
- discovery.seed_hosts=odfe-node1
- cluster.initial_master_nodes=odfe-node1
- bootstrap.memory_lock=true #
- "ES_JAVA_OPTS=-Xms512m -Xmx512m" #
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 65536 #
hard: 65536
volumes:
- odfe-data1:/usr/share/elasticsearch/data
ports:
- 9200:9200
- 9600:9600 # required for Performance Analyzer
networks:
- odfe-net
kibana:
image: amazon/opendistro-for-elasticsearch-kibana:1.13.2
container_name: odfe-kibana
ports:
- 5601:5601
expose:
- "5601"
environment:
ELASTICSEARCH_HOSTS: https://odfe-node1:9200
networks:
- odfe-net
logstash:
image: docker.elastic.co/logstash/logstash-oss:8.0.0-alpha2
container_name: logstash
volumes:
- type: bind
source: ./logstash_pipeline/
target: /usr/share/logstash/pipeline
read_only: true
ports:
- "5044:5044"
- "80:80"
networks:
- odfe-net
depends_on:
- odfe-node1
volumes:
odfe-data1:
networks:
odfe-net:
** Logstash confiugration**
input{
http_poller {
urls => {
authentication => {
method => get
url => "webapi/api/"
headers => {
"Content-Type" => "application/json"
}
}
}
request_timeout => 60
schedule => { every => "5s"}
codec => "json"
}
}
output {
opensearch {
hosts => ["https://odfe-node1:9200"]
index => "logstash-index-test"
user => "admin"
password => "admin"
ssl => true
cacert => "./logstash_pipeline/root-ca.pem"
ilm_enabled => false
}
}
can anybody help me with it?
thank you