Versions (relevant - OpenSearch/Dashboard/Server OS/Browser):
opensearchproject/opensearch latest
opensearchproject/opensearch-dashboards latest
Ubuntu 22.04
Describe the issue:
After configuring TLS security it appears that the Opensearch Dashboard cannot connect to Opensearch.
Configuration:
Docker-compose.yml
version: ‘3’
services:
opensearch-node1:
image: opensearchproject/opensearch:latest
container_name: node1
environment:
- cluster.name=opensearch-cluster # Name the cluster
- node.name=opensearch-node1 # Name the node that will run in this container
- plugins.security.allow_default_init_securityindex=true
- discovery.seed_hosts=opensearch-node1,opensearch-node2 # Nodes to look for when discovering the cluster
- cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2 # Nodes eligibile to serve as cluster manager
- bootstrap.memory_lock=true # Disable JVM heap memory swapping
- “OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m” # Set min and max JVM heap sizes to at least 50% of system RAM
- “DISABLE_INSTALL_DEMO_CONFIG=true” # Prevents execution of bundled demo script which installs demo certificates and security configurations to OpenSearch
- “DISABLE_SECURITY_PLUGIN=false” # Disables security plugin
ulimits:
memlock:
soft: -1 # Set memlock to unlimited (no soft or hard limit)
hard: -1
nofile:
soft: 65536 # Maximum number of open files for the opensearch user - set to at least 65536
hard: 65536
volumes:
- ./root-ca.pem:/usr/share/opensearch/config/root-ca.pem
- ./admin.pem:/usr/share/opensearch/config/admin.pem
- ./admin-key.pem:/usr/share/opensearch/config/admin-key.pem
- ./node1.pem:/usr/share/opensearch/config/node1.pem
- ./node1-key.pem:/usr/share/opensearch/config/node1-key.pem
- ./custom1-opensearch.yml:/usr/share/opensearch/config/opensearch.yml
- opensearch-data1:/usr/share/opensearch/data # Creates volume called opensearch-data1 and mounts it to the container
ports:
- 9200:9200 # REST API
- 9600:9600 # Performance Analyzer
networks:
- opensearch-net # All of the containers will join the same Docker bridge network
opensearch-node2:
image: opensearchproject/opensearch:latest
container_name: node2
environment:
- cluster.name=opensearch-cluster # Name the cluster
- node.name=opensearch-node2 # Name the node that will run in this container
- discovery.seed_hosts=opensearch-node1,opensearch-node2 # Nodes to look for when discovering the cluster
- cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2 # Nodes eligibile to serve as cluster manager
- plugins.security.allow_default_init_securityindex=true
- bootstrap.memory_lock=true # Disable JVM heap memory swapping
- “OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m” # Set min and max JVM heap sizes to at least 50% of system RAM
- “DISABLE_INSTALL_DEMO_CONFIG=true” # Prevents execution of bundled demo script which installs demo certificates and security configurations to OpenSearch
- “DISABLE_SECURITY_PLUGIN=false” # Disables security plugin
ulimits:
memlock:
soft: -1 # Set memlock to unlimited (no soft or hard limit)
hard: -1
nofile:
soft: 65536 # Maximum number of open files for the opensearch user - set to at least 65536
hard: 65536
volumes:
- ./root-ca.pem:/usr/share/opensearch/config/root-ca.pem
- ./admin.pem:/usr/share/opensearch/config/admin.pem
- ./admin-key.pem:/usr/share/opensearch/config/admin-key.pem
- ./node2.pem:/usr/share/opensearch/config/node2.pem
- ./node2-key.pem:/usr/share/opensearch/config/node2-key.pem
- ./custom2-opensearch.yml:/usr/share/opensearch/config/opensearch.yml
- opensearch-data2:/usr/share/opensearch/data # Creates volume called opensearch-data2 and mounts it to the container
networks:
- opensearch-net # All of the containers will join the same Docker bridge network
opensearch-dashboards:
image: opensearchproject/opensearch-dashboards:latest
container_name: opensearch-dashboards
ports:
- 5601:5601 # Map host port 5601 to container port 5601
expose:
- “5601” # Expose port 5601 for web access to OpenSearch Dashboards
environment:
- ‘OPENSEARCH_HOSTS=[“http://opensearch-node1:9200”,“http://opensearch-node2:9200”]’
- “DISABLE_SECURITY_DASHBOARDS_PLUGIN=true” # disables security dashboards plugin in OpenSearch Dashboards
networks:
- opensearch-net
volumes:
opensearch-data1:
opensearch-data2:
networks:
opensearch-net:
CUSTOM1-OPENSEARCH.YML - node 2 opensearch.yml is identical with the exception of node specific certs
plugins.security.ssl.transport.pemcert_filepath: node1.pem
plugins.security.ssl.transport.pemkey_filepath: node1-key.pem
plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.pem
plugins.security.ssl.transport.enforce_hostname_verification: false
plugins.security.ssl.http.enabled: true
plugins.security.ssl.http.pemcert_filepath: node1.pem
plugins.security.ssl.http.pemkey_filepath: node1-key.pem
plugins.security.ssl.http.pemtrustedcas_filepath: root-ca.pem
plugins.security.authcz.admin_dn:
- CN=SANITIZED,OU=Net,SANITIZED
plugins.security.nodes_dn: - 'CN=node1.dns.a-record,OU=Net,SANITIZED
- 'CN=node2.dns.a-record,OU=Net,SANITIZED
plugins.security.audit.type: internal_opensearch
plugins.security.enable_snapshot_restore_privilege: true
plugins.security.check_snapshot_restore_write_privileges: true
plugins.security.cache.ttl_minutes: 60
plugins.security.restapi.roles_enabled: [“all_access”, “security_rest_api_access”]
plugins.security.system_indices.enabled: true
plugins.security.system_indices.indices: [“.opendistro-alerting-config”, “.opendistro-alerting-alert*”, “.opendistro-anomaly-results*”, “.opendistro-anomaly-detector*”, “.opendistro-anomaly-checkpoints”>
node.max_local_storage_nodes: 3
Relevant Logs or Screenshots:
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:41:29Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:41:31Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:41:34Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
node1 | [2022-12-23T14:41:36,018][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-node1] cluster-manager not discovered or elected yet, an election requires a node with id [VVvHsQmpTpC37PKOOgGB7Q], have discovered [{opensearch-node1}{JGPmdl7wScCLJVD3eVHBJQ}{L6qZbWGeQlydNp221cTayw}{127.0.0.1}{127.0.0.1:9300}{dimr}{shard_indexing_pressure_enabled=true}] which is not a quorum; discovery will continue using [172.24.0.2:9300, 172.24.0.4:9300] from hosts providers and [{opensearch-node1}{JGPmdl7wScCLJVD3eVHBJQ}{L6qZbWGeQlydNp221cTayw}{127.0.0.1}{127.0.0.1:9300}{dimr}{shard_indexing_pressure_enabled=true}] from last-known cluster state; node term 4, last-accepted version 414 in term 4
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:41:36Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:41:39Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:41:41Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:41:44Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
node1 | [2022-12-23T14:41:46,022][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-node1] cluster-manager not discovered or elected yet, an election requires a node with id [VVvHsQmpTpC37PKOOgGB7Q], have discovered [{opensearch-node1}{JGPmdl7wScCLJVD3eVHBJQ}{L6qZbWGeQlydNp221cTayw}{127.0.0.1}{127.0.0.1:9300}{dimr}{shard_indexing_pressure_enabled=true}] which is not a quorum; discovery will continue using [172.24.0.2:9300, 172.24.0.4:9300] from hosts providers and [{opensearch-node1}{JGPmdl7wScCLJVD3eVHBJQ}{L6qZbWGeQlydNp221cTayw}{127.0.0.1}{127.0.0.1:9300}{dimr}{shard_indexing_pressure_enabled=true}] from last-known cluster state; node term 4, last-accepted version 414 in term 4
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:41:46Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:41:49Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:41:51Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:41:54Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
node1 | [2022-12-23T14:41:56,025][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-node1] cluster-manager not discovered or elected yet, an election requires a node with id [VVvHsQmpTpC37PKOOgGB7Q], have discovered [{opensearch-node1}{JGPmdl7wScCLJVD3eVHBJQ}{L6qZbWGeQlydNp221cTayw}{127.0.0.1}{127.0.0.1:9300}{dimr}{shard_indexing_pressure_enabled=true}] which is not a quorum; discovery will continue using [172.24.0.2:9300, 172.24.0.4:9300] from hosts providers and [{opensearch-node1}{JGPmdl7wScCLJVD3eVHBJQ}{L6qZbWGeQlydNp221cTayw}{127.0.0.1}{127.0.0.1:9300}{dimr}{shard_indexing_pressure_enabled=true}] from last-known cluster state; node term 4, last-accepted version 414 in term 4
node1 | [2022-12-23T14:41:56,028][WARN ][o.o.n.Node ] [opensearch-node1] timed out while waiting for initial discovery state - timeout: 30s
node1 | [2022-12-23T14:41:56,055][INFO ][o.o.h.AbstractHttpServerTransport] [opensearch-node1] publish_address {127.0.0.1:9200}, bound_addresses {127.0.0.1:9200}
node1 | [2022-12-23T14:41:56,055][INFO ][o.o.n.Node ] [opensearch-node1] started
node1 | [2022-12-23T14:41:56,056][INFO ][o.o.s.OpenSearchSecurityPlugin] [opensearch-node1] Node started
node1 | [2022-12-23T14:41:56,057][INFO ][o.o.s.c.ConfigurationRepository] [opensearch-node1] Will attempt to create index .opendistro_security and default configs if they are absent
node1 | [2022-12-23T14:41:56,059][INFO ][o.o.s.OpenSearchSecurityPlugin] [opensearch-node1] 0 OpenSearch Security modules loaded so far:
node1 | [2022-12-23T14:41:56,059][INFO ][o.o.s.c.ConfigurationRepository] [opensearch-node1] Background init thread started. Install default config?: true
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:41:56Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:41:59Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:42:01Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:42:04Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
node1 | [2022-12-23T14:42:06,028][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-node1] cluster-manager not discovered or elected yet, an election requires a node with id [VVvHsQmpTpC37PKOOgGB7Q], have discovered [{opensearch-node1}{JGPmdl7wScCLJVD3eVHBJQ}{L6qZbWGeQlydNp221cTayw}{127.0.0.1}{127.0.0.1:9300}{dimr}{shard_indexing_pressure_enabled=true}] which is not a quorum; discovery will continue using [172.24.0.2:9300, 172.24.0.4:9300] from hosts providers and [{opensearch-node1}{JGPmdl7wScCLJVD3eVHBJQ}{L6qZbWGeQlydNp221cTayw}{127.0.0.1}{127.0.0.1:9300}{dimr}{shard_indexing_pressure_enabled=true}] from last-known cluster state; node term 4, last-accepted version 414 in term 4
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:42:06Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:42:09Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:42:11Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:42:14Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
node1 | [2022-12-23T14:42:16,032][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-node1] cluster-manager not discovered or elected yet, an election requires a node with id [VVvHsQmpTpC37PKOOgGB7Q], have discovered [{opensearch-node1}{JGPmdl7wScCLJVD3eVHBJQ}{L6qZbWGeQlydNp221cTayw}{127.0.0.1}{127.0.0.1:9300}{dimr}{shard_indexing_pressure_enabled=true}] which is not a quorum; discovery will continue using [172.24.0.2:9300, 172.24.0.4:9300] from hosts providers and [{opensearch-node1}{JGPmdl7wScCLJVD3eVHBJQ}{L6qZbWGeQlydNp221cTayw}{127.0.0.1}{127.0.0.1:9300}{dimr}{shard_indexing_pressure_enabled=true}] from last-known cluster state; node term 4, last-accepted version 414 in term 4
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:42:16Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:42:19Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:42:21Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:42:24Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
node2 | [2022-12-23T14:42:25,625][INFO ][o.o.i.i.ManagedIndexCoordinator] [opensearch-node2] Performing move cluster state metadata.
node2 | [2022-12-23T14:42:25,627][INFO ][o.o.i.i.MetadataService ] [opensearch-node2] ISM config index not exist, so we cancel the metadata migration job.
node1 | [2022-12-23T14:42:26,046][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-node1] cluster-manager not discovered or elected yet, an election requires a node with id [VVvHsQmpTpC37PKOOgGB7Q], have discovered [{opensearch-node1}{JGPmdl7wScCLJVD3eVHBJQ}{L6qZbWGeQlydNp221cTayw}{127.0.0.1}{127.0.0.1:9300}{dimr}{shard_indexing_pressure_enabled=true}] which is not a quorum; discovery will continue using [172.24.0.2:9300, 172.24.0.4:9300] from hosts providers and [{opensearch-node1}{JGPmdl7wScCLJVD3eVHBJQ}{L6qZbWGeQlydNp221cTayw}{127.0.0.1}{127.0.0.1:9300}{dimr}{shard_indexing_pressure_enabled=true}] from last-known cluster state; node term 4, last-accepted version 414 in term 4
node1 | [2022-12-23T14:42:26,097][ERROR][o.o.s.c.ConfigurationRepository] [opensearch-node1] Cannot apply default config (this is maybe not an error!)
node1 | org.opensearch.discovery.ClusterManagerNotDiscoveredException: null
node1 | at org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction$AsyncSingleAction$2.onTimeout(TransportClusterManagerNodeAction.java:305) ~[opensearch-2.4.1.jar:2.4.1]
node1 | at org.opensearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:394) ~[opensearch-2.4.1.jar:2.4.1]
node1 | at org.opensearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onTimeout(ClusterStateObserver.java:294) ~[opensearch-2.4.1.jar:2.4.1]
node1 | at org.opensearch.cluster.service.ClusterApplierService$NotifyTimeout.run(ClusterApplierService.java:707) ~[opensearch-2.4.1.jar:2.4.1]
node1 | at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:747) ~[opensearch-2.4.1.jar:2.4.1]
node1 | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
node1 | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
node1 | at java.lang.Thread.run(Thread.java:833) [?:?]
node1 | [2022-12-23T14:42:26,119][ERROR][o.o.s.c.ConfigurationLoaderSecurity7] [opensearch-node1] Exception while retrieving configuration for [INTERNALUSERS, ACTIONGROUPS, CONFIG, ROLES, ROLESMAPPING, TENANTS, NODESDN, WHITELIST, ALLOWLIST, AUDIT] (index=.opendistro_security)
node1 | org.opensearch.cluster.block.ClusterBlockException: blocked by: [SERVICE_UNAVAILABLE/1/state not recovered / initialized];
node1 | at org.opensearch.cluster.block.ClusterBlocks.globalBlockedException(ClusterBlocks.java:205) ~[opensearch-2.4.1.jar:2.4.1]
node1 | at org.opensearch.cluster.block.ClusterBlocks.globalBlockedRaiseException(ClusterBlocks.java:191) ~[opensearch-2.4.1.jar:2.4.1]
node1 | at org.opensearch.action.get.TransportMultiGetAction.doExecute(TransportMultiGetAction.java:81) ~[opensearch-2.4.1.jar:2.4.1]
node1 | at org.opensearch.action.get.TransportMultiGetAction.doExecute(TransportMultiGetAction.java:58) ~[opensearch-2.4.1.jar:2.4.1]
node1 | at org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:218) [opensearch-2.4.1.jar:2.4.1]
node1 | at org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:118) [opensearch-index-management-2.4.1.0.jar:2.4.1.0]
node1 | at org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:216) [opensearch-2.4.1.jar:2.4.1]
node1 | at org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:232) [opensearch-security-2.4.1.0.jar:2.4.1.0]
node1 | at org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:149) [opensearch-security-2.4.1.0.jar:2.4.1.0]
node1 | at org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:216) [opensearch-2.4.1.jar:2.4.1]
node1 | at org.opensearch.performanceanalyzer.action.PerformanceAnalyzerActionFilter.apply(PerformanceAnalyzerActionFilter.java:78) [opensearch-performance-analyzer-2.4.1.0.jar:2.4.1.0]
node1 | at org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:216) [opensearch-2.4.1.jar:2.4.1]
node1 | at org.opensearch.action.support.TransportAction.execute(TransportAction.java:188) [opensearch-2.4.1.jar:2.4.1]
node1 | at org.opensearch.action.support.TransportAction.execute(TransportAction.java:107) [opensearch-2.4.1.jar:2.4.1]
node1 | at org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:110) [opensearch-2.4.1.jar:2.4.1]
node1 | at org.opensearch.client.node.NodeClient.doExecute(NodeClient.java:97) [opensearch-2.4.1.jar:2.4.1]
node1 | at org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:461) [opensearch-2.4.1.jar:2.4.1]
node1 | at org.opensearch.client.support.AbstractClient.multiGet(AbstractClient.java:577) [opensearch-2.4.1.jar:2.4.1]
node1 | at org.opensearch.security.configuration.ConfigurationLoaderSecurity7.loadAsync(ConfigurationLoaderSecurity7.java:208) [opensearch-security-2.4.1.0.jar:2.4.1.0]
node1 | at org.opensearch.security.configuration.ConfigurationLoaderSecurity7.load(ConfigurationLoaderSecurity7.java:99) [opensearch-security-2.4.1.0.jar:2.4.1.0]
node1 | at org.opensearch.security.configuration.ConfigurationRepository.getConfigurationsFromIndex(ConfigurationRepository.java:372) [opensearch-security-2.4.1.0.jar:2.4.1.0]
node1 | at org.opensearch.security.configuration.ConfigurationRepository.reloadConfiguration0(ConfigurationRepository.java:318) [opensearch-security-2.4.1.0.jar:2.4.1.0]
node1 | at org.opensearch.security.configuration.ConfigurationRepository.reloadConfiguration(ConfigurationRepository.java:303) [opensearch-security-2.4.1.0.jar:2.4.1.0]
node1 | at org.opensearch.security.configuration.ConfigurationRepository$1.run(ConfigurationRepository.java:163) [opensearch-security-2.4.1.0.jar:2.4.1.0]
node1 | at java.lang.Thread.run(Thread.java:833) [?:?]
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:42:26Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:42:29Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:42:31Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:42:34Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}
node1 | [2022-12-23T14:42:36,048][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-node1] cluster-manager not discovered or elected yet, an election requires a node with id [VVvHsQmpTpC37PKOOgGB7Q], have discovered [{opensearch-node1}{JGPmdl7wScCLJVD3eVHBJQ}{L6qZbWGeQlydNp221cTayw}{127.0.0.1}{127.0.0.1:9300}{dimr}{shard_indexing_pressure_enabled=true}] which is not a quorum; discovery will continue using [172.24.0.2:9300, 172.24.0.4:9300] from hosts providers and [{opensearch-node1}{JGPmdl7wScCLJVD3eVHBJQ}{L6qZbWGeQlydNp221cTayw}{127.0.0.1}{127.0.0.1:9300}{dimr}{shard_indexing_pressure_enabled=true}] from last-known cluster state; node term 4, last-accepted version 414 in term 4
opensearch-dashboards | {“type”:“log”,“@timestamp”:“2022-12-23T14:42:36Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ConnectionError]: connect ECONNREFUSED 172.24.0.2:9200”}