When I create a detector with the CloudTrail rules on an index with CloudTrail events, I get an error on every detection interval.
[2023-04-05T21:52:37,018][DEBUG][o.o.a.s.TransportSearchAction] [828cc5bff286a5406738409ca83391b1] [u_8qP75iTK66k0B_sIDHlw][.opensearch-sap-cloudtrail-detectors-queries-000001][0]: Failed to execute [SearchRequest{searchType=QUERY_THEN_FETCH, indices=[.opensearch-sap-cloudtrail-detectors-queries-000001], indicesOptions=IndicesOptions[ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, expand_wildcards_closed=false, expand_wildcards_hidden=false, allow_aliases_to_multiple_indices=true, forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true], routing='null', preference='null', requestCache=null, scroll=null, maxConcurrentShardRequests=0, batchedReduceSize=512, preFilterShardSize=null, allowPartialSearchResults=true, localClusterAlias=null, getOrCreateAbsoluteStartMillis=-1, ccsMinimizeRoundtrips=true, source={"query":{"bool":{"filter":[{"match":{"index":{"query":"aws-cloudtrail-o-abc123abc1-2023-04-00","operator":"OR","prefix_length":0,"max_expansions":50,"fuzzy_transpositions":true,"lenient":false,"zero_terms_query":"NONE","auto_generate_synonyms_phrase_query":true,"boost":1.0}}},{"match":{"monitor_id":{"query":"L6dgU4cB3ZlzOPpO7w-8","operator":"OR","prefix_length":0,"max_expansions":50,"fuzzy_transpositions":true,"lenient":false,"zero_terms_query":"NONE","auto_generate_synonyms_phrase_query":true,"boost":1.0}}},{"percolate_ext":{"field":"query","documents":[{"eventID_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"ad1927a2-1f65-4097-8c9c-e3b8cb8e89c3","awsRegion_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"us-east-1","eventCategory_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"Management","eventVersion_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"1.08","responseElements_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":null,"sourceIPAddress_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"cloudtrail.amazonaws.com","eventSource_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"s3.amazonaws.com","requestParameters_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":{"bucketName":"aws-controltower-logs-123451234512-us-east-1","Host":"aws-controltower-logs-123451234512-us-east-1.s3.us-east-1.amazonaws.com","acl":""},"resources_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":[{"accountId":"123451234512","type":"AWS__IP__S__IP__ucket","ARN":"arn:aws:s__IP__:aws-controltower-logs-123451234512-us-east-1"}],"userAgent_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"cloudtrail.amazonaws.com","readOnly_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":true,"userIdentity_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":{"type":"AWSService","invokedBy":"cloudtrail.amazonaws.com"},"eventType_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"AwsApiCall","additionalEventData_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":{"SignatureVersion":"SigV4","CipherSuite":"ECDHE-RSA-AES128-GCM-SHA256","bytesTransferredIn":0,"AuthenticationMethod":"AuthHeader","x-amz-id-2":"iU488rfzqL5D0FucrT6KhluceTsmqal9+wLHw7i7WEt2IsXqp71DtIjN1mpwDe2I0sxhwOqyUj8=","bytesTransferredOut":572},"sharedEventID_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"a4ebeb91-ddef-4842-bb7a-7dcbb4c9161d","requestID_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"7PQWE8386Y9PW0EX","eventTime_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"2023-04-05T21:48:40Z","eventName_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"GetBucketAcl","recipientAccountId_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"123451234512","managementEvent_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":true},{"eventID_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"694d054f-2c36-4862-b380-5368a6910719","awsRegion_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"us-east-1","eventCategory_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"Management","eventVersion_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"1.08","responseElements_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":null,"sourceIPAddress_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"AWS __PATH__[__PATH__],"userAgent_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"AWS __PATH__ Internal"},"eventType_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"AwsApiCall","sharedEventID_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"affa188c-1049-4c3a-af2f-382b50547efc","requestID_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"5a06cc5e-88ed-4ce6-8475-981850b4529f","eventTime_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"2023-04-05T21:48:45Z","eventName_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"Decrypt","recipientAccountId_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"123451234512","managementEvent_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":true},{"eventID_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"528ee5ed-bfc5-421a-9d35-dea947da6a45","awsRegion_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"us-east-1","eventCategory_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"Management","eventVersion_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"1.08","responseElements_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":null,"sourceIPAddress_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"cloudtrail.amazonaws.com","eventSource_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":"s3.amazonaws.com","requestParameters_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":{"bucketName":"aws-controltower-logs-123451234512-us-east-1","Host":"aws-controltower-logs-123451234512-us-east-1.s3.us-east-1.amazonaws.com","acl":""},"resources_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8":[{"accountId":"123451234512","type":"AWS__IP__S__IP__ucket","ARN":"arn:aws:s__IP__:aws-controltower-logs-123451234512-us-east-1"}]__PATH__
RemoteTransportException[[c484db26552bedcb9522c8447baa6f89][__IP__][__PATH__[__PATH__]]]; nested: QueryShardException[failed to create query: Inconsistency of field data structures across documents for field [requestParameters_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8.limit] of doc [68]. index options: expected 'NONE', but it has 'DOCS_AND_FREQS_AND_POSITIONS'.]; nested: IllegalArgumentException[Inconsistency of field data structures across documents for field [requestParameters_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8.limit] of doc [68]. index options: expected 'NONE', but it has 'DOCS_AND_FREQS_AND_POSITIONS'.];
Caused by: [__PATH__] QueryShardException[failed to create query: Inconsistency of field data structures across documents for field [requestParameters_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8.limit] of doc [68]. index options: expected 'NONE', but it has 'DOCS_AND_FREQS_AND_POSITIONS'.]; nested: IllegalArgumentException[Inconsistency of field data structures across documents for field [requestParameters_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8.limit] of doc [68]. index options: expected 'NONE', but it has 'DOCS_AND_FREQS_AND_POSITIONS'.];
at org.opensearch.index.query.QueryShardContext.toQuery(QueryShardContext.java:482)
at org.opensearch.index.query.QueryShardContext.toQuery(QueryShardContext.java:465)
at org.opensearch.search.SearchService.parseSource(SearchService.java:1233)
at org.opensearch.search.SearchService.createContext(SearchService.java:982)
at org.opensearch.search.SearchService.executeQueryPhase(SearchService.java:590)
at org.opensearch.search.SearchService$2.lambda$onResponse$0(SearchService.java:563)
at org.opensearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:73)
at org.opensearch.action.ActionRunnable$2.doRun(ActionRunnable.java:88)
at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52)
at org.opensearch.threadpool.TaskAwareRunnable.doRun(TaskAwareRunnable.java:78)
at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52)
at org.opensearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:59)
at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:815)
at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.lang.Thread.run(Thread.java:829)
Caused by: java.lang.IllegalArgumentException: Inconsistency of field data structures across documents for field [requestParameters_aws-cloudtrail-o-abc123abc1-2023-04-00_L6dgU4cB3ZlzOPpO7w-8.limit] of doc [68]. index options: expected 'NONE', but it has 'DOCS_AND_FREQS_AND_POSITIONS'.
at org.apache.lucene.index.IndexingChain$FieldSchema.raiseNotSame(IndexingChain.java:1325)
at org.apache.lucene.index.IndexingChain$FieldSchema.assertSame(IndexingChain.java:1320)
at org.apache.lucene.index.IndexingChain$FieldSchema.assertSameSchema(IndexingChain.java:1405)
at org.apache.lucene.index.IndexingChain.processDocument(IndexingChain.java:558)
at org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:241)
at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:432)
at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1533)
at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1504)
at org.opensearch.percolator.PercolateQueryBuilderExt.createMultiDocumentSearcher(PercolateQueryBuilderExt.java:578)
at org.opensearch.percolator.PercolateQueryBuilderExt.doToQuery(PercolateQueryBuilderExt.java:536)
at org.opensearch.index.query.AbstractQueryBuilder.toQuery(AbstractQueryBuilder.java:116)
at org.opensearch.index.query.BoolQueryBuilder.addBooleanClauses(BoolQueryBuilder.java:346)
at org.opensearch.index.query.BoolQueryBuilder.doToQuery(BoolQueryBuilder.java:329)
at org.opensearch.index.query.AbstractQueryBuilder.toQuery(AbstractQueryBuilder.java:116)
at org.opensearch.index.query.QueryShardContext.lambda$toQuery$3(QueryShardContext.java:466)
at org.opensearch.index.query.QueryShardContext.toQuery(QueryShardContext.java:478)
... 16 more
I increased the mapping limit to 10k for “.opensearch-sap-*” using an index template, but it didn’t change anything. This is the configuration of “.opensearch-sap-cloudtrail-detectors-queries-000001”.
{
".opensearch-sap-cloudtrail-detectors-queries-000001": {
"aliases": {
".opensearch-sap-cloudtrail-detectors-queries": {}
},
"mappings": {
"_meta": {
"schema_version": 1
},
"properties": {
...
}
},
"settings": {
"index": {
"mapping": {
"total_fields": {
"limit": "10000"
}
},
"hidden": "true",
"number_of_shards": "5",
"provided_name": ".opensearch-sap-cloudtrail-detectors-queries-000001",
"creation_date": "1680731074565",
"analysis": {
"analyzer": {
"rule_analyzer": {
"char_filter": [
"rule_ws_filter"
],
"tokenizer": "keyword"
}
},
"char_filter": {
"rule_ws_filter": {
"pattern": "(_ws_)",
"type": "pattern_replace",
"replacement": " "
}
}
},
"number_of_replicas": "1",
"uuid": "5DBEP_xbQMqlvNoom_TfXA",
"version": {
"created": "136267827"
}
}
}
}
}
I noticed that the “.opensearch-sap-cloudtrail-detectors-queries-000001” index has about 2800 mappings and the data index has almost 4000, but I don’t know if that’s related.