Versions (relevant - OpenSearch/Dashboard/Server OS/Browser):
opensearch 3.0.0
debian 11
Describe the issue:
I have migrated from 2.19 to 3.0.0 opensearch version.
S3 snapshots were OK.
After migration, I got the following exception.
Configuration:
Relevant Logs or Screenshots:
[2025-06-20T17:03:36,992][WARN ][r.suppressed ] [****] path: /_snapshot/xx-s3-repository, params: {repository=xx-s3-repository}
org.opensearch.repositories.RepositoryVerificationException: [xx-s3-repository] path is not accessible on cluster-manager node
at org.opensearch.repositories.blobstore.BlobStoreRepository.startVerification(BlobStoreRepository.java:2956) ~[opensearch-3.0.0.jar:3.0.0]
at org.opensearch.repositories.RepositoriesService$3.doRun(RepositoriesService.java:381) ~[opensearch-3.0.0.jar:3.0.0]
at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:975) ~[opensearch-3.0.0.jar:3.0.0]
at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52) ~[opensearch-3.0.0.jar:3.0.0]
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]
at java.base/java.lang.Thread.run(Thread.java:1583) [?:?]
Caused by: java.io.IOException: Unable to upload object [tests-j_3z_BoSRNuTsnqIeID3cw/master.dat] using a single upload
at org.opensearch.repositories.s3.S3BlobContainer.executeSingleUpload(S3BlobContainer.java:558) ~[?:?]
at org.opensearch.repositories.s3.S3BlobContainer.lambda$writeBlobWithMetadata$1(S3BlobContainer.java:198) ~[?:?]
at java.base/java.security.AccessController.doPrivileged(AccessController.java:571) ~[?:?]
at org.opensearch.repositories.s3.SocketAccess.doPrivilegedIOException(SocketAccess.java:62) ~[?:?]
at org.opensearch.repositories.s3.S3BlobContainer.writeBlobWithMetadata(S3BlobContainer.java:196) ~[?:?]
at org.opensearch.repositories.s3.S3BlobContainer.writeBlob(S3BlobContainer.java:180) ~[?:?]
at org.opensearch.repositories.s3.S3BlobContainer.writeBlobAtomic(S3BlobContainer.java:373) ~[?:?]
at org.opensearch.repositories.blobstore.BlobStoreRepository.startVerification(BlobStoreRepository.java:2948) ~[opensearch-3.0.0.jar:3.0.0]
… 6 more
Caused by: software.amazon.awssdk.services.s3.model.S3Exception: The Content-SHA256 you specified did not match what we received (Service: S3, Status Code: 400, Request ID: ac10c908:196517e2361:d02b6:aff, Extended Request ID: dba7831b6072cf203a84b4e0addf4b0709ce7e02a31e284dd21adf225d6176bd) (SDK Attempt Count: 1)
at software.amazon.awssdk.services.s3.model.S3Exception$BuilderImpl.build(S3Exception.java:113) ~[?:?]
at software.amazon.awssdk.services.s3.model.S3Exception$BuilderImpl.build(S3Exception.java:61) ~[?:?]
at software.amazon.awssdk.core.internal.http.pipeline.stages.utils.RetryableStageHelper.retryPolicyDisallowedRetryException(RetryableStageHelper.java:168) ~[?:?]
at software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:73) ~[?:?]
at software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:36) ~[?:?]
at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206) ~[?:?]
at software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:53) ~[?:?]
at software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:35) ~[?:?]
at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.executeWithTimer(ApiCallTimeoutTrackingStage.java:82) ~[?:?]
at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:62) ~[?:?]
at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:43) ~[?:?]
at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage.execute(ApiCallMetricCollectionStage.java:50) ~[?:?]
at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage.execute(ApiCallMetricCollectionStage.java:32) ~[?:?]
at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206) ~[?:?]
at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206) ~[?:?]
at software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:37) ~[?:?]
at software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:26) ~[?:?]
at software.amazon.awssdk.core.internal.http.AmazonSyncHttpClient$RequestExecutionBuilderImpl.execute(AmazonSyncHttpClient.java:210) ~[?:?]
at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.invoke(BaseSyncClientHandler.java:103) ~[?:?]
at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.doExecute(BaseSyncClientHandler.java:173) ~[?:?]
at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.lambda$execute$1(BaseSyncClientHandler.java:80) ~[?:?]
at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.measureApiCallSuccess(BaseSyncClientHandler.java:182) ~[?:?]
at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.execute(BaseSyncClientHandler.java:74) ~[?:?]
at software.amazon.awssdk.core.client.handler.SdkSyncClientHandler.execute(SdkSyncClientHandler.java:45) ~[?:?]
at software.amazon.awssdk.awscore.client.handler.AwsSyncClientHandler.execute(AwsSyncClientHandler.java:53) ~[?:?]
at software.amazon.awssdk.services.s3.DefaultS3Client.putObject(DefaultS3Client.java:11202) ~[?:?]
at org.opensearch.repositories.s3.S3BlobContainer.lambda$executeSingleUpload$24(S3BlobContainer.java:555) ~[?:?]
at org.opensearch.repositories.s3.SocketAccess.lambda$doPrivilegedVoid$0(SocketAccess.java:71) ~[?:?]
at java.base/java.security.AccessController.doPrivileged(AccessController.java:319) ~[?:?]
at org.opensearch.repositories.s3.SocketAccess.doPrivilegedVoid(SocketAccess.java:70) ~[?:?]
at org.opensearch.repositories.s3.S3BlobContainer.executeSingleUpload(S3BlobContainer.java:554) ~[?:?]
at org.opensearch.repositories.s3.S3BlobContainer.lambda$writeBlobWithMetadata$1(S3BlobContainer.java:198) ~[?:?]
at java.base/java.security.AccessController.doPrivileged(AccessController.java:571) ~[?:?]
at org.opensearch.repositories.s3.SocketAccess.doPrivilegedIOException(SocketAccess.java:62) ~[?:?]
at org.opensearch.repositories.s3.S3BlobContainer.writeBlobWithMetadata(S3BlobContainer.java:196) ~[?:?]
at org.opensearch.repositories.s3.S3BlobContainer.writeBlob(S3BlobContainer.java:180) ~[?:?]
at org.opensearch.repositories.s3.S3BlobContainer.writeBlobAtomic(S3BlobContainer.java:373) ~[?:?]
at org.opensearch.repositories.blobstore.BlobStoreRepository.startVerification(BlobStoreRepository.java:2948) ~[opensearch-3.0.0.jar:3.0.0]
… 6 more