Hello,
since 4 Weeks I try to change my webshop-search-server from ES to Opensearch.
Installed Opensearch 2.01 via docker-compose + Dashboard and connect it via the PHP client.
All works fine but after some days (~3-4 days) I get the error “all shards failed” and have to restart my 2x dockers…
Where can I search for my problem? The query and Index works all time, but… after 3-4 days the whole docker crashes…
docker-compose logs output:
at org.opensearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:644) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:362) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:679) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:459) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.action.search.AbstractSearchAsyncAction$1.onFailure(AbstractSearchAsyncAction.java:293) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.action.search.SearchExecutionStatsCollector.onFailure(SearchExecutionStatsCollector.java:104) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:74) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:615) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.transport.TransportService$6.handleException(TransportService.java:750) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.security.transport.SecurityInterceptor$RestoringTransportResponseHandler.handleException(SecurityInterceptor.java:318) [opensearch-security-2.0.1.0.jar:2.0.1.0]
node1 | at org.opensearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1370) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.transport.TransportService$DirectResponseChannel.processException(TransportService.java:1484) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.transport.TransportService$DirectResponseChannel.sendResponse(TransportService.java:1458) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.transport.TransportService.sendLocalRequest(TransportService.java:982) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.transport.TransportService$3.sendRequest(TransportService.java:152) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.transport.TransportService.sendRequestInternal(TransportService.java:884) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:212) [opensearch-security-2.0.1.0.jar:2.0.1.0]
node1 | at org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:665) [opensearch-security-2.0.1.0.jar:2.0.1.0]
node1 | at org.opensearch.transport.TransportService.sendRequest(TransportService.java:771) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:846) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:834) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:198) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:133) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:280) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:314) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:251) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:417) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:217) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:1006) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:769) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:410) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:80) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:138) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:103) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:499) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:284) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:125) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:204) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:118) [opensearch-index-management-2.0.1.0.jar:2.0.1.0]
node1 | at org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:202) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:325) [opensearch-security-2.0.1.0.jar:2.0.1.0]
node1 | at org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:157) [opensearch-security-2.0.1.0.jar:2.0.1.0]
node1 | at org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:202) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.performanceanalyzer.action.PerformanceAnalyzerActionFilter.apply(PerformanceAnalyzerActionFilter.java:78) [opensearch-performance-analyzer-2.0.1.0.jar:2.0.1.0]
node1 | at org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:202) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.action.support.TransportAction.execute(TransportAction.java:174) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.action.support.TransportAction.execute(TransportAction.java:102) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:110) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:106) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:423) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:135) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:125) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:128) [opensearch-security-2.0.1.0.jar:2.0.1.0]
node1 | at org.opensearch.rest.RestController.dispatchRequest(RestController.java:311) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.rest.RestController.tryAllHandlers(RestController.java:397) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.rest.RestController.dispatchRequest(RestController.java:240) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) [opensearch-security-2.0.1.0.jar:2.0.1.0]
node1 | at org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:366) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:445) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:356) [opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) [transport-netty4-client-2.0.1.jar:2.0.1]
node1 | at org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) [transport-netty4-client-2.0.1.jar:2.0.1]
node1 | at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) [transport-netty4-client-2.0.1.jar:2.0.1]
node1 | at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) [netty-codec-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) [netty-codec-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) [netty-codec-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:327) [netty-codec-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:299) [netty-codec-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) [netty-handler-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) [netty-codec-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) [netty-handler-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) [netty-handler-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) [netty-handler-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:510) [netty-codec-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:449) [netty-codec-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:279) [netty-codec-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) [netty-transport-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) [netty-common-4.1.73.Final.jar:4.1.73.Final]
node1 | at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.73.Final.jar:4.1.73.Final]
node1 | at java.lang.Thread.run(Thread.java:833) [?:?]
node1 | Caused by: org.opensearch.OpenSearchException: java.io.OptionalDataException
node1 | at org.opensearch.security.support.Base64Helper.deserializeObject(Base64Helper.java:185) ~[?:?]
node1 | at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:155) ~[?:?]
node1 | at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) ~[?:?]
node1 | at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:651) ~[?:?]
node1 | at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:118) ~[?:?]
node1 | at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:43) ~[?:?]
node1 | at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:103) ~[opensearch-2.0.1.jar:2.0.1]
node1 | at org.opensearch.transport.TransportService.sendLocalRequest(TransportService.java:946) ~[opensearch-2.0.1.jar:2.0.1]
node1 | ... 102 more```