Hi,
I have one opensearch cluster (3 nodes) I installed from rpm. It works fine.
Now I’m trying to install new opensearch cluster (3 nodes) using docker. (lab environment)
I took the same files (security, certs, config).
My docker-compose.yaml
version: '3'
services:
opensearch-0:
image: opensearchproject/opensearch:1.3.3
container_name: opensearch-0
environment:
- cluster.name=cluster1
- node.name=opensearch-0
- discovery.seed_hosts=opensearch-0.cluster.local,opensearch-1.cluster.local,opensearch-2.cluster.local
- cluster.initial_master_nodes=opensearch-0.cluster.local,opensearch-1.cluster.local,opensearch-2.cluster.local
- "OPENSEARCH_JAVA_OPTS=-Xms4g -Xmx4g"
- "DISABLE_INSTALL_DEMO_CONFIG=true"
- "DISABLE_SECURITY_PLUGIN=false"
- network.host=172.16.16.141
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 65536
hard: 65536
volumes:
- ./opensearch/configs/log4j2.properties:/usr/share/opensearch/config/log4j2.properties
- ./opensearch/configs/opensearch.yml:/usr/share/opensearch/config/opensearch.yml
- ./opensearch/configs/opensearch.pem:/usr/share/opensearch/config/opensearch.pem
- ./opensearch/configs/opensearch.key:/usr/share/opensearch/config/opensearch.key
- ./opensearch/configs/root-ca.pem:/usr/share/opensearch/config/root-ca.pem
- ./opensearch/configs/admin.pem:/usr/share/opensearch/config/admin.pem
- ./opensearch/configs/admin.key:/usr/share/opensearch/config/admin.key
- ./opensearch/configs/opensearch-security/action_groups.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/action_groups.yml
- ./opensearch/configs/opensearch-security/audit.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/audit.yml
- ./opensearch/configs/opensearch-security/config.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/config.yml
- ./opensearch/configs/opensearch-security/internal_users.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/internal_users.yml
- ./opensearch/configs/opensearch-security/nodes_dn.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/nodes_dn.yml
- ./opensearch/configs/opensearch-security/roles_mapping.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/roles_mapping.yml
- ./opensearch/configs/opensearch-security/roles.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/roles.yml
- ./opensearch/configs/opensearch-security/tenants.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/tenants.yml
- ./opensearch/configs/opensearch-security/whitelist.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/whitelist.yml
- /opensearch:/usr/share/opensearch/data
network_mode: "host"
extra_hosts:
- "opensearch-0.cluster.local:172.16.16.141"
- "opensearch-1.cluster.local:172.16.16.142"
- "opensearch-2.cluster.local:172.16.16.143"
When I started (docker-compose up) I got an error
opensearch-0 | {"type": "server", "timestamp": "2022-09-22T14:54:30,982Z", "level": "WARN", "component": "o.o.c.c.ClusterFormationFailureHelper", "cluster.name": "cluster1", "node.name": "opensearch-0", "message": "master not discovere
d yet, this node has not previously joined a bootstrapped (v7+) cluster, and this node must discover master-eligible nodes [opensearch-0.cluster.local, opensearch-1.cluster.local, opensearch-2.cluster.local] to bootstrap a cluster: have d
iscovered [{opensearch-0}{iqgzTq-_T_uJK8f1Z6voKA}{TBYTqgt_SaSA5zbfD6jz5w}{172.16.16.141}{172.16.16.141:9300}{dimr}{shard_indexing_pressure_enabled=true}]; discovery will continue using [172.16.16.142:9300, 172.16.16.143:9300] from hosts p
roviders and [{opensearch-0}{iqgzTq-_T_uJK8f1Z6voKA}{TBYTqgt_SaSA5zbfD6jz5w}{172.16.16.141}{172.16.16.141:9300}{dimr}{shard_indexing_pressure_enabled=true}] from last-known cluster state; node term 0, last-accepted version 0 in term 0" }
opensearch-0 | {"type": "server", "timestamp": "2022-09-22T14:54:31,006Z", "level": "ERROR", "component": "o.o.s.c.ConfigurationRepository", "cluster.name": "cluster1", "node.name": "opensearch-0", "message": "Cannot apply default conf
ig (this is maybe not an error!)",
opensearch-0 | "stacktrace": ["org.opensearch.discovery.MasterNotDiscoveredException: null",
opensearch-0 | "at org.opensearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$2.onTimeout(TransportMasterNodeAction.java:275) ~[opensearch-1.3.3.jar:1.3.3]",
opensearch-0 | "at org.opensearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:369) ~[opensearch-1.3.3.jar:1.3.3]",
opensearch-0 | "at org.opensearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onTimeout(ClusterStateObserver.java:287) ~[opensearch-1.3.3.jar:1.3.3]",
opensearch-0 | "at org.opensearch.cluster.service.ClusterApplierService$NotifyTimeout.run(ClusterApplierService.java:692) ~[opensearch-1.3.3.jar:1.3.3]",
opensearch-0 | "at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:733) ~[opensearch-1.3.3.jar:1.3.3]",
opensearch-0 | "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
opensearch-0 | "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
opensearch-0 | "at java.lang.Thread.run(Thread.java:829) [?:?]"] }
opensearch-0 | {"type": "server", "timestamp": "2022-09-22T14:54:31,020Z", "level": "ERROR", "component": "o.o.s.c.ConfigurationLoaderSecurity7", "cluster.name": "cluster1", "node.name": "opensearch-0", "message": "Exception while retr
ieving configuration for [INTERNALUSERS, ACTIONGROUPS, CONFIG, ROLES, ROLESMAPPING, TENANTS, NODESDN, WHITELIST, AUDIT] (index=.opendistro_security)",
opensearch-0 | "stacktrace": ["org.opensearch.cluster.block.ClusterBlockException: blocked by: [SERVICE_UNAVAILABLE/1/state not recovered / initialized];",
opensearch-0 | "at org.opensearch.cluster.block.ClusterBlocks.globalBlockedException(ClusterBlocks.java:202) ~[opensearch-1.3.3.jar:1.3.3]",
opensearch-0 | "at org.opensearch.cluster.block.ClusterBlocks.globalBlockedRaiseException(ClusterBlocks.java:188) ~[opensearch-1.3.3.jar:1.3.3]",
opensearch-0 | "at org.opensearch.action.get.TransportMultiGetAction.doExecute(TransportMultiGetAction.java:76) ~[opensearch-1.3.3.jar:1.3.3]",
opensearch-0 | "at org.opensearch.action.get.TransportMultiGetAction.doExecute(TransportMultiGetAction.java:53) ~[opensearch-1.3.3.jar:1.3.3]",
opensearch-0 | "at org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) [opensearch-1.3.3.jar:1.3.3]",
opensearch-0 | "at org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:120) [opensearch-index-management-1.3.3.0.jar:1.3.3.0]",
opensearch-0 | "at org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) [opensearch-1.3.3.jar:1.3.3]",
opensearch-0 | "at org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:234) [opensearch-security-1.3.3.0.jar:1.3.3.0]",
opensearch-0 | "at org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) [opensearch-security-1.3.3.0.jar:1.3.3.0]",
opensearch-0 | "at org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) [opensearch-1.3.3.jar:1.3.3]",
opensearch-0 | "at org.opensearch.performanceanalyzer.action.PerformanceAnalyzerActionFilter.apply(PerformanceAnalyzerActionFilter.java:78) [opensearch-performance-analyzer-1.3.3.0.jar:1.3.3.0]",
opensearch-0 | "at org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) [opensearch-1.3.3.jar:1.3.3]",
opensearch-0 | "at org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) [opensearch-1.3.3.jar:1.3.3]",
opensearch-0 | "at org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) [opensearch-1.3.3.jar:1.3.3]",
opensearch-0 | "at org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) [opensearch-1.3.3.jar:1.3.3]",
opensearch-0 | "at org.opensearch.client.node.NodeClient.doExecute(NodeClient.java:95) [opensearch-1.3.3.jar:1.3.3]",
opensearch-0 | "at org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) [opensearch-1.3.3.jar:1.3.3]",
opensearch-0 | "at org.opensearch.client.support.AbstractClient.multiGet(AbstractClient.java:554) [opensearch-1.3.3.jar:1.3.3]",
opensearch-0 | "at org.opensearch.security.configuration.ConfigurationLoaderSecurity7.loadAsync(ConfigurationLoaderSecurity7.java:211) [opensearch-security-1.3.3.0.jar:1.3.3.0]",
opensearch-0 | "at org.opensearch.security.configuration.ConfigurationLoaderSecurity7.load(ConfigurationLoaderSecurity7.java:102) [opensearch-security-1.3.3.0.jar:1.3.3.0]",
opensearch-0 | "at org.opensearch.security.configuration.ConfigurationRepository.getConfigurationsFromIndex(ConfigurationRepository.java:375) [opensearch-security-1.3.3.0.jar:1.3.3.0]",
opensearch-0 | "at org.opensearch.security.configuration.ConfigurationRepository.reloadConfiguration0(ConfigurationRepository.java:321) [opensearch-security-1.3.3.0.jar:1.3.3.0]",
opensearch-0 | "at org.opensearch.security.configuration.ConfigurationRepository.reloadConfiguration(ConfigurationRepository.java:306) [opensearch-security-1.3.3.0.jar:1.3.3.0]",
opensearch-0 | "at org.opensearch.security.configuration.ConfigurationRepository$1.run(ConfigurationRepository.java:166) [opensearch-security-1.3.3.0.jar:1.3.3.0]",
opensearch-0 | "at java.lang.Thread.run(Thread.java:829) [?:?]"] }
Maybe someone can help. I don’t know why security settings didn’t apply.