I am using Logstash version 6.8.23 to sync data to OpenSearch 1.2.0 indices
In the logstash output plugin for elastic search used in the docker configuration file is as shown below
RUN apt-get install -y wget; wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add - \
&& apt-get install -y apt-transport-https \
&& echo "deb https://artifacts.elastic.co/packages/6.x/apt stable main" | tee -a /etc/apt/sources.list.d/elastic-6.x.list \
&& apt-get update \
&& apt-get install -y logstash=1:6.8.23-1 \
&& /usr/share/logstash/bin/logstash-plugin install --version '10.8.6' logstash-output-elasticsearch
Input plugin in cofig file :
jdbc {
tags => "staff"
jdbc_connection_string => "jdbc:postgresql://${DB_HOST}:${DB_PORT}/${DB_NAME}"
jdbc_user => "${DB_USER}"
jdbc_password => "${DB_PASSWORD}"
# schedule => "* * * * *"
jdbc_driver_class => "org.postgresql.Driver"
statement_filepath => "/staffboard-logstash/queries/stm-staff-query.sql"
use_column_value => true
tracking_column => "last_updated"
tracking_column_type => "timestamp"
last_run_metadata_path => "/usr/share/logstash/logstash_jdbc_last_run_stm_staff_profile"
jdbc_paging_enabled => true
jdbc_page_size => 10000
jdbc_fetch_size => 10000
}
Filter plugin :
if "staff" in [tags] {
json {
source => "sns_approval_status"
target => "sns_approval_status"
}
}
Output plugin
if "staff" in [tags] {
stdout {
codec => json
}
elasticsearch {
hosts => ["${ELASTIC_HOST}"]
ssl => false
ssl_certificate_verification => false
index => "index-v33"
document_id => "%{sb_id}"
user => "${ELASTIC_USER_NAME}"
password => "${ELASTIC_PASSWORD}"
ilm_enabled => false
manage_template => true
template => "/staffboard-logstash/mappings/stm/stm-staff-profile-index.json"
template_name => "index-test-33"
template_overwrite => true
}
}
Configuration file :
{
"template": "index-v33",
"version": 1,
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},
"@version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"account_type": {
"type": "long"
},
"age_range": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"bio": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"body_characteristics": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"branch_cd": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"branch_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"can_display_age": {
"type": "boolean"
},
"can_display_bio": {
"type": "boolean"
},
"can_display_body_characteristics": {
"type": "boolean"
},
"can_display_body_shape": {
"type": "boolean"
},
"can_display_date_of_birth": {
"type": "boolean"
},
"can_display_fav_styling": {
"type": "boolean"
},
"can_display_front_label": {
"type": "boolean"
},
"can_display_gender": {
"type": "boolean"
},
"can_display_have_child": {
"type": "boolean"
},
"can_display_interest": {
"type": "boolean"
},
"can_display_personal_color": {
"type": "boolean"
},
"can_display_profile_on_st": {
"type": "boolean"
},
"can_display_shoe_size": {
"type": "boolean"
},
"can_display_shoulder_stature": {
"type": "boolean"
},
"can_display_size": {
"type": "boolean"
},
"can_display_store": {
"type": "boolean"
},
"can_display_user_bottom_size": {
"type": "boolean"
},
"can_display_user_height": {
"type": "boolean"
},
"can_display_user_name": {
"type": "boolean"
},
"can_display_user_top_size": {
"type": "boolean"
},
"can_display_user_weight": {
"type": "boolean"
},
"coubic_url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"created_at": {
"type": "date"
},
"crowned_staff": {
"type": "boolean"
},
"csvideo_count": {
"type": "long"
},
"cumulation_count": {
"type": "long"
},
"dob": {
"type": "date"
},
"fav_styling_1": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"fav_styling_2": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"fav_styling_3": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"follower_count": {
"type": "long"
},
"front_display_label_code": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"front_display_label_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"group_id": {
"type": "long"
},
"group_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"have_child": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"interests": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"is_post_approval_required": {
"type": "boolean"
},
"label_code": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"label_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"label_name_kana": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"last_updated": {
"type": "date"
},
"last_updated_stm": {
"type": "date"
},
"personal_color": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"role_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"sb_id": {
"type": "long"
},
"sb_permission_id": {
"type": "long"
},
"sb_role_id": {
"type": "long"
},
"shop_code": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"shop_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"shop_name_kana": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"shop_number": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"shoulder_stature": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"sns_approval_status": {
"type": "nested",
"properties": {
"approval_status": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"sns_handle": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"sns_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"status": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"store_account_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"store_prefecture_code": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"store_visit": {
"type": "boolean"
},
"styling_count": {
"type": "long"
},
"tags": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_body_shape": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_bottom_size": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_code": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_gender": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_height": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_image_url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_name_kana": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_nickname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_shoe_size": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_top_size": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_weight": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"weight_range": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
While I run logstash the template is created, however the data is not getting inserted into the index
The error thrown is as shown below
[logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>[“index”, {:_id=>“120838”, :_index=>“stm-staff-profile-index-v45”, :_routing=>nil, :_type=>“doc”}, #LogStash::Event:0x7f035b7d], :response=>{“index”=>{“_index”=>“index-v33”, “_type”=>“doc”, “_id”=>“120838”, “status”=>400, “error”=>{“type”=>“illegal_argument_exception”, “reason”=>“Rejecting mapping update to [index-v33] as the final mapping would have more than 1 type: [_doc, doc]”}}}}
Any help is appreciated.
Thanks in advance