Versions:
Opensearch: 2.19.2
Describe the issue:
Can’t remove read_only from index:
PUT /.ds-ds-bb-api-restrito-000647/_settings
{
"blocks": {
"write": false,
"read_only": false
}
}
{
"error": {
"root_cause": [
{
"type": "cluster_block_exception",
"reason": "index [.ds-ds-bb-api-restrito-000647] blocked by: [FORBIDDEN/5/index read-only (api)];"
}
],
"type": "cluster_block_exception",
"reason": "index [.ds-ds-bb-api-restrito-000647] blocked by: [FORBIDDEN/5/index read-only (api)];"
},
"status": 403
}
Trying to delete it doesn’t do any better
DELETE /.ds-ds-bb-api-restrito-000647?error_trace=true
{
"error": {
"root_cause": [
{
"type": "cluster_block_exception",
"reason": "index [.ds-ds-bb-api-restrito-000647] blocked by: [FORBIDDEN/5/index read-only (api)];",
"stack_trace": "ClusterBlockException[index [.ds-ds-bb-api-restrito-000647] blocked by: [FORBIDDEN/5/index read-only (api)];]\n\tat org.opensearch.cluster.block.ClusterBlocks.indicesAllowReleaseResources(ClusterBlocks.java:284)\n\tat org.opensearch.action.admin.indices.delete.TransportDeleteIndexAction.checkBlock(TransportDeleteIndexAction.java:114)\n\tat org.opensearch.action.admin.indices.delete.TransportDeleteIndexAction.checkBlock(TransportDeleteIndexAction.java:66)\n\tat org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction$AsyncSingleAction.checkForBlock(TransportClusterManagerNodeAction.java:471)\n\tat org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction$AsyncSingleAction.doStart(TransportClusterManagerNodeAction.java:276)\n\tat org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction$AsyncSingleAction.tryAction(TransportClusterManagerNodeAction.java:245)\n\tat org.opensearch.action.support.RetryableAction$1.doRun(RetryableAction.java:139)\n\tat org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52)\n\tat org.opensearch.common.util.concurrent.OpenSearchExecutors$DirectExecutorService.execute(OpenSearchExecutors.java:343)\n\tat org.opensearch.action.support.RetryableAction.run(RetryableAction.java:117)\n\tat org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction.doExecute(TransportClusterManagerNodeAction.java:206)\n\tat org.opensearch.action.admin.indices.delete.TransportDeleteIndexAction.doExecute(TransportDeleteIndexAction.java:109)\n\tat org.opensearch.action.admin.indices.delete.TransportDeleteIndexAction.doExecute(TransportDeleteIndexAction.java:66)\n\tat org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:220)\n\tat org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:118)\n\tat org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:218)\n\tat org.opensearch.indexmanagement.controlcenter.notification.filter.IndexOperationActionFilter.apply(IndexOperationActionFilter.kt:39)\n\tat org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:218)\n\tat org.opensearch.performanceanalyzer.action.PerformanceAnalyzerActionFilter.apply(PerformanceAnalyzerActionFilter.java:81)\n\tat org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:218)\n\tat org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:391)\n\tat org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:166)\n\tat org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:218)\n\tat org.opensearch.action.support.TransportAction.execute(TransportAction.java:190)\n\tat org.opensearch.action.support.HandledTransportAction$TransportHandler.messageReceived(HandledTransportAction.java:133)\n\tat org.opensearch.action.support.HandledTransportAction$TransportHandler.messageReceived(HandledTransportAction.java:129)\n\tat org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:207)\n\tat org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:314)\n\tat org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:155)\n\tat org.opensearch.security.OpenSearchSecurityPlugin$6$1.messageReceived(OpenSearchSecurityPlugin.java:873)\n\tat org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:113)\n\tat org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:44)\n\tat org.opensearch.performanceanalyzer.transport.RTFPerformanceAnalyzerTransportRequestHandler.messageReceived(RTFPerformanceAnalyzerTransportRequestHandler.java:63)\n\tat org.opensearch.wlm.WorkloadManagementTransportInterceptor$RequestHandler.messageReceived(WorkloadManagementTransportInterceptor.java:63)\n\tat org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:108)\n\tat org.opensearch.transport.NativeMessageHandler.handleRequest(NativeMessageHandler.java:278)\n\tat org.opensearch.transport.NativeMessageHandler.handleMessage(NativeMessageHandler.java:146)\n\tat org.opensearch.transport.NativeMessageHandler.messageReceived(NativeMessageHandler.java:126)\n\tat org.opensearch.transport.InboundHandler.messageReceivedFromPipeline(InboundHandler.java:120)\n\tat org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:112)\n\tat org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:768)\n\tat org.opensearch.transport.InboundBytesHandler.forwardFragments(InboundBytesHandler.java:137)\n\tat org.opensearch.transport.InboundBytesHandler.doHandleBytes(InboundBytesHandler.java:77)\n\tat org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:124)\n\tat org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:113)\n\tat org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:95)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)\n\tat io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)\n\tat io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:107)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)\n\tat io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1515)\n\tat io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1378)\n\tat io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1427)\n\tat io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530)\n\tat io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469)\n\tat io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)\n\tat io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)\n\tat io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)\n\tat io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:697)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:660)\n\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)\n\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n\tat java.lang.Thread.run(Thread.java:1583)\n"
}
],
"type": "cluster_block_exception",
"reason": "index [.ds-ds-bb-api-restrito-000647] blocked by: [FORBIDDEN/5/index read-only (api)];",
"stack_trace": "ClusterBlockException[index [.ds-ds-bb-api-restrito-000647] blocked by: [FORBIDDEN/5/index read-only (api)];]\n\tat org.opensearch.cluster.block.ClusterBlocks.indicesAllowReleaseResources(ClusterBlocks.java:284)\n\tat org.opensearch.action.admin.indices.delete.TransportDeleteIndexAction.checkBlock(TransportDeleteIndexAction.java:114)\n\tat org.opensearch.action.admin.indices.delete.TransportDeleteIndexAction.checkBlock(TransportDeleteIndexAction.java:66)\n\tat org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction$AsyncSingleAction.checkForBlock(TransportClusterManagerNodeAction.java:471)\n\tat org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction$AsyncSingleAction.doStart(TransportClusterManagerNodeAction.java:276)\n\tat org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction$AsyncSingleAction.tryAction(TransportClusterManagerNodeAction.java:245)\n\tat org.opensearch.action.support.RetryableAction$1.doRun(RetryableAction.java:139)\n\tat org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52)\n\tat org.opensearch.common.util.concurrent.OpenSearchExecutors$DirectExecutorService.execute(OpenSearchExecutors.java:343)\n\tat org.opensearch.action.support.RetryableAction.run(RetryableAction.java:117)\n\tat org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction.doExecute(TransportClusterManagerNodeAction.java:206)\n\tat org.opensearch.action.admin.indices.delete.TransportDeleteIndexAction.doExecute(TransportDeleteIndexAction.java:109)\n\tat org.opensearch.action.admin.indices.delete.TransportDeleteIndexAction.doExecute(TransportDeleteIndexAction.java:66)\n\tat org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:220)\n\tat org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:118)\n\tat org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:218)\n\tat org.opensearch.indexmanagement.controlcenter.notification.filter.IndexOperationActionFilter.apply(IndexOperationActionFilter.kt:39)\n\tat org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:218)\n\tat org.opensearch.performanceanalyzer.action.PerformanceAnalyzerActionFilter.apply(PerformanceAnalyzerActionFilter.java:81)\n\tat org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:218)\n\tat org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:391)\n\tat org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:166)\n\tat org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:218)\n\tat org.opensearch.action.support.TransportAction.execute(TransportAction.java:190)\n\tat org.opensearch.action.support.HandledTransportAction$TransportHandler.messageReceived(HandledTransportAction.java:133)\n\tat org.opensearch.action.support.HandledTransportAction$TransportHandler.messageReceived(HandledTransportAction.java:129)\n\tat org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:207)\n\tat org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:314)\n\tat org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:155)\n\tat org.opensearch.security.OpenSearchSecurityPlugin$6$1.messageReceived(OpenSearchSecurityPlugin.java:873)\n\tat org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:113)\n\tat org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:44)\n\tat org.opensearch.performanceanalyzer.transport.RTFPerformanceAnalyzerTransportRequestHandler.messageReceived(RTFPerformanceAnalyzerTransportRequestHandler.java:63)\n\tat org.opensearch.wlm.WorkloadManagementTransportInterceptor$RequestHandler.messageReceived(WorkloadManagementTransportInterceptor.java:63)\n\tat org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:108)\n\tat org.opensearch.transport.NativeMessageHandler.handleRequest(NativeMessageHandler.java:278)\n\tat org.opensearch.transport.NativeMessageHandler.handleMessage(NativeMessageHandler.java:146)\n\tat org.opensearch.transport.NativeMessageHandler.messageReceived(NativeMessageHandler.java:126)\n\tat org.opensearch.transport.InboundHandler.messageReceivedFromPipeline(InboundHandler.java:120)\n\tat org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:112)\n\tat org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:768)\n\tat org.opensearch.transport.InboundBytesHandler.forwardFragments(InboundBytesHandler.java:137)\n\tat org.opensearch.transport.InboundBytesHandler.doHandleBytes(InboundBytesHandler.java:77)\n\tat org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:124)\n\tat org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:113)\n\tat org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:95)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)\n\tat io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)\n\tat io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:107)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)\n\tat io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1515)\n\tat io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1378)\n\tat io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1427)\n\tat io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530)\n\tat io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469)\n\tat io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)\n\tat io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)\n\tat io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)\n\tat io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:697)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:660)\n\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)\n\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n\tat java.lang.Thread.run(Thread.java:1583)\n"
},
"status": 403
}
Configuration:
GET /.ds-ds-bb-api-restrito-000647/_settings
{
".ds-ds-bb-api-restrito-000647": {
"settings": {
"index": {
"replication": {
"type": "DOCUMENT"
},
"refresh_interval": "5s",
"hidden": "true",
"number_of_shards": "1",
"blocks": {
"read_only": "true",
"write": "true"
},
"provided_name": ".ds-ds-bb-api-restrito-000647",
"creation_date": "1760767561377",
"number_of_replicas": "1",
"uuid": "0h6CdeI0SXKdYCdiKROnkQ",
"version": {
"created": "136408027"
}
}
}
}
}
User role:
GET /_plugins/_security/api/roles/cluster_admin
{
"cluster_admin": {
"reserved": false,
"hidden": false,
"cluster_permissions": [
"cluster_composite_ops_ro",
"alerting_full_access",
"alerting_read_access",
"alerting_ack_alerts",
"cluster_all"
],
"index_permissions": [],
"tenant_permissions": [
{
"tenant_patterns": [
"alerts_tenant"
],
"allowed_actions": [
"kibana_all_write"
]
}
],
"static": false
}
}
Found only a elastic forum post talking abount lack of disk. I tried increasing all watermarks but the error kept the same
Relevant Logs or Screenshots:
