Hi Folks,
I am facing this weird issue - I am not able to map Geopoint and would really need someone’s help
Here is my template
{
"index_patterns" : "newdata-*",
"version" : 60001,
"settings" : {
"index.refresh_interval" : "5s",
"number_of_shards": 1
},
"mappings" : {
"dynamic_templates" : [ {
"message_field" : {
"path_match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text",
"norms" : false
}
}
}, {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text", "norms" : false,
"fields" : {
"keyword" : { "type": "keyword", "ignore_above": 256 }
}
}
}
} ],
"properties" : {
"@timestamp": { "type": "date"},
"@version": { "type": "keyword"},
"geoip" : {
"dynamic": true,
"properties" : {
"source_ip": { "type": "ip" },
"location" : { "type" : "geo_point" },
"latitude" : { "type" : "half_float" },
"longitude" : { "type" : "half_float" }
}
}
}
}
}
And here is my parsed json data
{
"_index": "newdata-2021.02.27",
"_type": "_doc",
"_id": "i6qB4XcBfPcjg3oYsyMt",
"_version": 1,
"_score": null,
"_source": {
"date_end": "2020-03-04T05:29:59",
"geoip": {
"ip": "209.85.220.41",
"country_code2": "US",
"latitude": 37.751,
"country_name": "United States",
"asn": 15169,
"longitude": -97.822,
"timezone": "America/Chicago",
"continent_code": "NA",
"location": {
"lat": 37.751,
"lon": -97.822
},
"as_org": "Google LLC",
"country_code3": "US"
},
"org_name": "google.com",
"message": "{\"org_name\": \"google.com\", \"policy_spf\": \"pass\", \"org_email\": \"noreply-dmarc-support@google.com\", \"policy_dkim\": \"pass\", \"policy_pct\": \"100\", \"auth_spf_result\": \"pass\", \"auth_dkim_domain\": \"google.com\", \"auth_dkim_result\": \"pass\", \"identifier_header_from\": \"test.com\", \"date_end\": \"2020-03-04T05:29:59\", \"date_start\": \"2020-03-03T05:30:00\", \"source_ip\": \"209.85.220.41\", \"count\": 2, \"auth_spf_domain\": \"test.com\", \"policy_p\": \"reject\", \"submitter\": \"google.com\", \"policy_disposition\": \"none\", \"policy_domain\": \"test.com\", \"id\": \"2468918998957341147\"}",
"id": "2468918998957341147",
"path": "/opt/nedaa/log/test.json",
"auth_dkim_result": "pass",
"@timestamp": "2021-02-27T03:21:48.936Z",
"dmarc_align": "True",
"type": "json",
"policy_disposition": "none",
"@version": "1",
"count": 2,
"policy_dkim": "pass",
"policy_spf": "pass",
"submitter": "google.com",
"policy_domain": "test.com",
"is_from_spf": "YES",
"policy_pct": "100",
"org_email": "noreply-dmarc-support@google.com",
"auth_spf_result": "pass",
"identifier_header_from": "test.com",
"host": "dmarc365",
"date_start": "2020-03-03T05:30:00",
"auth_spf_domain": "test.com",
"policy_p": "reject",
"auth_dkim_domain": "google.com",
"source_ip": "209.85.220.41"
},
"fields": {
"date_start": [
"2020-03-03T05:30:00.000Z"
],
"@timestamp": [
"2021-02-27T03:21:48.936Z"
],
"date_end": [
"2020-03-04T05:29:59.000Z"
]
},
"sort": [
1614396108936
]
}
My logstash parsing
# Add geo coordinates / ASN info / IP rep.
if [source_ip] {
geoip {
cache_size => 10000
source => "source_ip"
database => "/usr/share/elasticsearch/modules/ingest-geoip/GeoLite2-City.mmdb"
}
geoip {
cache_size => 10000
source => "source_ip"
database => "/usr/share/elasticsearch/modules/ingest-geoip/GeoLite2-ASN.mmdb"
}
}
}
output {
elasticsearch {
hosts => ["https://127.0.0.1:9200"]
user => admin
password => admin
ssl => true
ssl_certificate_verification => false
template => "/etc/logstash/elastic-template-7x.json"
ilm_enabled => false
index => "newdata-%{+YYYY.MM.dd}"
# document_type => "doc"
}
}
The index pattern newdata-* does not contain any of the following compatible field types: geo_point
And still there is this error