Versions:
v 1.0.0
Amazon OpenSearch Service
Managed Cluster
Describe the issue:
I currently have a search that fires every five minutes via an AWS lambda that makes a request to the _search endpoint for my index. This search has been working successfully for four months, but suddenly today it has started giving me the following error:
Text fields are not optimised for operations that require per-document field data like aggregations and sorting, so these operations are disabled by default. Please use a keyword field instead. Alternatively, set fielddata=true on [webaclId] in order to load field data by uninverting the inverted index. Note that this can use significant memory.
This error only happens when I run the search via my lambda. When I run the same search using the dev tools in the dashboard it runs just fine.
As best as I can tell, I’ve configured webaclId as a keyword field, not a text field, as per my mapping shared below, so I can’t figure out the issue. Has anyone else experienced this or found a solution?
The search is as follows:
{
"query": {
"constant_score": {
"filter": {
"bool": {
"must": [
{
"wildcard": {
"rateBasedRuleList.limitKey": "*"
}
},
{
"match_phrase": {
"nonTerminatingMatchingRules.action": "COUNT"
}
},
{
"match_phrase": {
"terminatingRuleId": "Default_Action"
}
},
{
"range": {
"timestamp": {
"gte": "now-1h"
}
}
}
]
}
}
}
},
"aggs": {
"webaclId_count": {
"terms": {
"field": "webaclId"
},
"aggs": {
"single_webaclId_count": {
"value_count": {
"field": "webaclId"
}
},
"limit": {
"bucket_selector": {
"buckets_path": {
"count": "single_webaclId_count"
},
"script": "params.count > 100"
}
}
}
}
}
}
The relevant section of the mapping template is as follows:
{
"myindex" : {
"mappings" : {
"properties" : {
"nonTerminatingMatchingRules" : {
"properties" : {
"action" : {
"type" : "keyword"
},
"ruleId" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"ruleMatchDetails" : {
"properties" : {
"conditionType" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"location" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"matchedData" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"sensitivityLevel" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
}
}
},
"rateBasedRuleList" : {
"properties" : {
"limitKey" : {
"type" : "keyword"
},
"limitValue" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"maxRateAllowed" : {
"type" : "long"
},
"rateBasedRuleId" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"rateBasedRuleName" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"terminatingRuleId" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"timestamp" : {
"type" : "date"
},
"webaclId" : {
"type" : "keyword"
}
}
}
}
}
Full error:
{
"error": {
"root_cause": [
{
"type": "illegal_argument_exception",
"reason": "Text fields are not optimised for operations that require per-document field data like aggregations and sorting, so these operations are disabled by default. Please use a keyword field instead. Alternatively, set fielddata=true on [webaclId] in order to load field data by uninverting the inverted index. Note that this can use significant memory."
}
],
"type": "search_phase_execution_exception",
"reason": "",
"phase": "fetch",
"grouped": true,
"failed_shards": [
{
"shard": 0,
"index": "redacted",
"node": "redacted",
"reason": {
"type": "illegal_argument_exception",
"reason": "Text fields are not optimised for operations that require per-document field data like aggregations and sorting, so these operations are disabled by default. Please use a keyword field instead. Alternatively, set fielddata=true on [webaclId] in order to load field data by uninverting the inverted index. Note that this can use significant memory."
}
}
],
"caused_by": {
"type": "null_pointer_exception",
"reason": null
}
},
"status": 400
}
