Running 5 node opendistro version 0.9.0 ES cluster on AWS EC2 instance (8 core, EBS volumes ext4, Java11). There are about 1k indices having time-series data and about 1TB of data already ingested. All indices are having most of their fields with keyword analyser. ES service is running under docker container.
Since very long time we are observing constant CPU spikes generated through management threads of security plugin. This is hampering indexing and query performance. (CPU usage is staying approximately around 80-90%)
Can someone shed lights on what could be wrong in setup? Following is the stack trace captured through hotthreads api:
89.3% (446.4ms out of 500ms) cpu usage by thread 'elasticsearch[myes1][management][T#2]'
2/10 snapshots sharing following 35 elements
java.base@11.0.1/sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:145)
java.base@11.0.1/sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
java.base@11.0.1/java.nio.file.Files.readAttributes(Files.java:1763)
java.base@11.0.1/java.nio.file.Files.size(Files.java:2380)
app//org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243)
app//org.elasticsearch.index.store.ByteSizeCachingDirectory.estimateSizeInBytes(ByteSizeCachingDirectory.java:56)
app//org.elasticsearch.index.store.ByteSizeCachingDirectory.access$200(ByteSizeCachingDirectory.java:37)
app//org.elasticsearch.index.store.ByteSizeCachingDirectory$1.refresh(ByteSizeCachingDirectory.java:90)
app//org.elasticsearch.index.store.ByteSizeCachingDirectory$1.refresh(ByteSizeCachingDirectory.java:72)
app//org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54)
app//org.elasticsearch.index.store.ByteSizeCachingDirectory.estimateSizeInBytes(ByteSizeCachingDirectory.java:121)
app//org.elasticsearch.index.store.Store$StoreDirectory.estimateSize(Store.java:712)
app//org.elasticsearch.index.store.Store.stats(Store.java:359)
app//org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:1035)
app//org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:180)
app//org.elasticsearch.indices.IndicesService.indexShardStats(IndicesService.java:403)
app//org.elasticsearch.indices.IndicesService.statsByShard(IndicesService.java:357)
app//org.elasticsearch.indices.IndicesService.stats(IndicesService.java:348)
app//org.elasticsearch.node.NodeService.stats(NodeService.java:109)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:74)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:39)
app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:138)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:259)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:255)
com.amazon.opendistroforelasticsearch.security.ssl.transport.OpenDistroSecuritySSLRequestHandler.messageReceivedDecorate(OpenDistroSecuritySSLRequestHandler.java:194)
com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityRequestHandler.messageReceivedDecorate(OpenDistroSecurityRequestHandler.java:163)
com.amazon.opendistroforelasticsearch.security.ssl.transport.OpenDistroSecuritySSLRequestHandler.messageReceived(OpenDistroSecuritySSLRequestHandler.java:116)
com.amazon.opendistroforelasticsearch.security.OpenDistroSecurityPlugin$7$1.messageReceived(OpenDistroSecurityPlugin.java:645)
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
app//org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:686)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@11.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.1/java.lang.Thread.run(Thread.java:834)
2/10 snapshots sharing following 40 elements
app//org.elasticsearch.bootstrap.ESPolicy.implies(ESPolicy.java:102)
java.base@11.0.1/java.security.ProtectionDomain.implies(ProtectionDomain.java:321)
java.base@11.0.1/java.security.ProtectionDomain.impliesWithAltFilePerm(ProtectionDomain.java:353)
java.base@11.0.1/java.security.AccessControlContext.checkPermission(AccessControlContext.java:450)
java.base@11.0.1/java.security.AccessController.checkPermission(AccessController.java:895)
java.base@11.0.1/java.lang.SecurityManager.checkPermission(SecurityManager.java:322)
java.base@11.0.1/java.lang.SecurityManager.checkRead(SecurityManager.java:661)
java.base@11.0.1/sun.nio.fs.UnixPath.checkRead(UnixPath.java:818)
java.base@11.0.1/sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
java.base@11.0.1/sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:145)
java.base@11.0.1/sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
java.base@11.0.1/java.nio.file.Files.readAttributes(Files.java:1763)
java.base@11.0.1/java.nio.file.Files.getLastModifiedTime(Files.java:2314)
app//org.elasticsearch.index.translog.BaseTranslogReader.getLastModifiedTime(BaseTranslogReader.java:147)
app//org.elasticsearch.index.translog.Translog.findEarliestLastModifiedAge(Translog.java:435)
app//org.elasticsearch.index.translog.Translog.earliestLastModifiedAge(Translog.java:423)
app//org.elasticsearch.index.translog.Translog.stats(Translog.java:861)
app//org.elasticsearch.index.engine.InternalEngine.getTranslogStats(InternalEngine.java:545)
app//org.elasticsearch.index.shard.IndexShard.translogStats(IndexShard.java:1065)
app//org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:216)
app//org.elasticsearch.indices.IndicesService.indexShardStats(IndicesService.java:403)
app//org.elasticsearch.indices.IndicesService.statsByShard(IndicesService.java:357)
app//org.elasticsearch.indices.IndicesService.stats(IndicesService.java:348)
app//org.elasticsearch.node.NodeService.stats(NodeService.java:109)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:74)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:39)
app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:138)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:259)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:255)
com.amazon.opendistroforelasticsearch.security.ssl.transport.OpenDistroSecuritySSLRequestHandler.messageReceivedDecorate(OpenDistroSecuritySSLRequestHandler.java:194)
com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityRequestHandler.messageReceivedDecorate(OpenDistroSecurityRequestHandler.java:163)
com.amazon.opendistroforelasticsearch.security.ssl.transport.OpenDistroSecuritySSLRequestHandler.messageReceived(OpenDistroSecuritySSLRequestHandler.java:116)
com.amazon.opendistroforelasticsearch.security.OpenDistroSecurityPlugin$7$1.messageReceived(OpenDistroSecurityPlugin.java:645)
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
app//org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:686)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@11.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.1/java.lang.Thread.run(Thread.java:834)
6/10 snapshots sharing following 20 elements
app//org.elasticsearch.indices.IndicesService.indexShardStats(IndicesService.java:403)
app//org.elasticsearch.indices.IndicesService.statsByShard(IndicesService.java:357)
app//org.elasticsearch.indices.IndicesService.stats(IndicesService.java:348)
app//org.elasticsearch.node.NodeService.stats(NodeService.java:109)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:74)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:39)
app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:138)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:259)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:255)
com.amazon.opendistroforelasticsearch.security.ssl.transport.OpenDistroSecuritySSLRequestHandler.messageReceivedDecorate(OpenDistroSecuritySSLRequestHandler.java:194)
com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityRequestHandler.messageReceivedDecorate(OpenDistroSecurityRequestHandler.java:163)
com.amazon.opendistroforelasticsearch.security.ssl.transport.OpenDistroSecuritySSLRequestHandler.messageReceived(OpenDistroSecuritySSLRequestHandler.java:116)
com.amazon.opendistroforelasticsearch.security.OpenDistroSecurityPlugin$7$1.messageReceived(OpenDistroSecurityPlugin.java:645)
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
app//org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:686)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@11.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.1/java.lang.Thread.run(Thread.java:834)
::: {myes5}{Qr38QOK4ToyEjeaO7mq_gA}{zWo0A1pcSVW1nEecnsC5Wg}{10.126.1.105}{10.126.1.105:9300}
Hot threads at 2019-12-09T13:06:00.157Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
100.2% (501.2ms out of 500ms) cpu usage by thread 'elasticsearch[myes5][management][T#1]'
8/10 snapshots sharing following 22 elements
app//org.elasticsearch.index.shard.IndexShard.completionStats(IndexShard.java:1071)
app//org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:210)
app//org.elasticsearch.indices.IndicesService.indexShardStats(IndicesService.java:403)
app//org.elasticsearch.indices.IndicesService.statsByShard(IndicesService.java:357)
app//org.elasticsearch.indices.IndicesService.stats(IndicesService.java:348)
app//org.elasticsearch.node.NodeService.stats(NodeService.java:109)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:74)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:39)
app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:138)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:259)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:255)
com.amazon.opendistroforelasticsearch.security.ssl.transport.OpenDistroSecuritySSLRequestHandler.messageReceivedDecorate(OpenDistroSecuritySSLRequestHandler.java:194)
com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityRequestHandler.messageReceivedDecorate(OpenDistroSecurityRequestHandler.java:284)
com.amazon.opendistroforelasticsearch.security.ssl.transport.OpenDistroSecuritySSLRequestHandler.messageReceived(OpenDistroSecuritySSLRequestHandler.java:166)
com.amazon.opendistroforelasticsearch.security.OpenDistroSecurityPlugin$7$1.messageReceived(OpenDistroSecurityPlugin.java:645)
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
app//org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1087)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@11.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.1/java.lang.Thread.run(Thread.java:834)
2/10 snapshots sharing following 39 elements
java.base@11.0.1/java.security.ProtectionDomain.implies(ProtectionDomain.java:321)
java.base@11.0.1/java.security.ProtectionDomain.impliesWithAltFilePerm(ProtectionDomain.java:353)
java.base@11.0.1/java.security.AccessControlContext.checkPermission(AccessControlContext.java:450)
java.base@11.0.1/java.security.AccessController.checkPermission(AccessController.java:895)
java.base@11.0.1/java.lang.SecurityManager.checkPermission(SecurityManager.java:322)
java.base@11.0.1/java.lang.SecurityManager.checkRead(SecurityManager.java:661)
java.base@11.0.1/sun.nio.fs.UnixPath.checkRead(UnixPath.java:818)
java.base@11.0.1/sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
java.base@11.0.1/sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:145)
java.base@11.0.1/sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
java.base@11.0.1/java.nio.file.Files.readAttributes(Files.java:1763)
java.base@11.0.1/java.nio.file.Files.getLastModifiedTime(Files.java:2314)
app//org.elasticsearch.index.translog.BaseTranslogReader.getLastModifiedTime(BaseTranslogReader.java:147)
app//org.elasticsearch.index.translog.Translog.findEarliestLastModifiedAge(Translog.java:437)
app//org.elasticsearch.index.translog.Translog.earliestLastModifiedAge(Translog.java:423)
app//org.elasticsearch.index.translog.Translog.stats(Translog.java:861)
app//org.elasticsearch.index.engine.InternalEngine.getTranslogStats(InternalEngine.java:545)
app//org.elasticsearch.index.shard.IndexShard.translogStats(IndexShard.java:1065)
app//org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:216)
app//org.elasticsearch.indices.IndicesService.indexShardStats(IndicesService.java:403)
app//org.elasticsearch.indices.IndicesService.statsByShard(IndicesService.java:357)
app//org.elasticsearch.indices.IndicesService.stats(IndicesService.java:348)
app//org.elasticsearch.node.NodeService.stats(NodeService.java:109)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:74)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:39)
app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:138)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:259)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:255)
com.amazon.opendistroforelasticsearch.security.ssl.transport.OpenDistroSecuritySSLRequestHandler.messageReceivedDecorate(OpenDistroSecuritySSLRequestHandler.java:194)
com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityRequestHandler.messageReceivedDecorate(OpenDistroSecurityRequestHandler.java:284)
com.amazon.opendistroforelasticsearch.security.ssl.transport.OpenDistroSecuritySSLRequestHandler.messageReceived(OpenDistroSecuritySSLRequestHandler.java:166)
com.amazon.opendistroforelasticsearch.security.OpenDistroSecurityPlugin$7$1.messageReceived(OpenDistroSecurityPlugin.java:645)
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
app//org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1087)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@11.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.1/java.lang.Thread.run(Thread.java:834)
Note: We had disabled performance analyser plugin as it was causing the similar CPU spikes.