Versions (relevant - OpenSearch/Dashboard/Server OS/Browser):
OpenSearch 2.15
Describe the issue:
A Boolean value that specifies whether to ignore malformed values when running a query on a derived field.
From the derived field document
Does this mean the ignore_malfomed only support query?
How about fields parameter in the search request?
It would be great to ignore the error for fetching derived field to search response too, I think.
Reproduce example requests.
###
DELETE /derived-sample-logs
### create index
PUT /derived-sample-logs
{
"mappings": {
"properties": {
"request": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"clientip": {
"type": "keyword"
}
}
}
}
### PUT the derived fields
PUT /derived-sample-logs/_mapping
{
"derived": {
"timestamp": {
"type": "date",
"format": "MM/dd/yyyy HH:mm:ss",
"script": {
"source": "emit(Long.parseLong(doc[\"request.keyword\"].value.splitOnToken(\" \")[0]))"
}
},
"method": {
"type": "keyword",
"script": {
"source": "emit(doc[\"request.keyword\"].value.splitOnToken(\" \")[1])"
}
},
"size": {
"type": "long",
"script": {
"source": "emit(Long.parseLong(doc[\"request.keyword\"].value.splitOnToken(\" \")[5]))"
}
}
}
}
### index an error data
PUT /derived-sample-logs/_doc/7
{
"request": "hoge GET /english/images/france98_venues.gif HTTP/1.0 200 778",
"clientip": "61.177.2.0"
}
### match_all search with fields param
GET /derived-sample-logs/_search
{
"fields": ["request", "clientip", "timestamp"]
}
The error from OpenSearch.
{
"error": {
"root_cause": [
{
"type": "script_exception",
"reason": "runtime error",
"script_stack": [
"java.base/java.lang.NumberFormatException.forInputString(NumberFormatException.java:67)",
"java.base/java.lang.Long.parseLong(Long.java:709)",
"java.base/java.lang.Long.parseLong(Long.java:832)",
"emit(Long.parseLong(doc[\"request.keyword\"].value.splitOnToken(\" \")[0]))",
" ^---- HERE"
],
"script": "emit(Long.parseLong(doc[\"request.keyword\"].value.splitOnToken(\" \")[0]))",
"lang": "painless",
"position": {
"offset": 66,
"start": 0,
"end": 71
}
}
],
"type": "search_phase_execution_exception",
"reason": "all shards failed",
"phase": "query",
"grouped": true,
"failed_shards": [
{
"shard": 0,
"index": "derived-sample-logs",
"node": "kcP6GWKmR_6oA1IGf3d4fw",
"reason": {
"type": "script_exception",
"reason": "runtime error",
"script_stack": [
"java.base/java.lang.NumberFormatException.forInputString(NumberFormatException.java:67)",
"java.base/java.lang.Long.parseLong(Long.java:709)",
"java.base/java.lang.Long.parseLong(Long.java:832)",
"emit(Long.parseLong(doc[\"request.keyword\"].value.splitOnToken(\" \")[0]))",
" ^---- HERE"
],
"script": "emit(Long.parseLong(doc[\"request.keyword\"].value.splitOnToken(\" \")[0]))",
"lang": "painless",
"position": {
"offset": 66,
"start": 0,
"end": 71
},
"caused_by": {
"type": "number_format_exception",
"reason": "For input string: \"hoge\""
}
}
}
]
},
"status": 400
}