Versions (relevant - OpenSearch/Dashboard/Server OS/Browser):
Hi I am using this versions
amazon/opendistro-for-elasticsearch:1.13.3
amazon/opendistro-for-elasticsearch-kibana:1.13.2
opensearchproject/logstash-oss-with-opensearch-output-plugin:7.16.2
i am running in docker using azure VM’s
like i want elasticsearch run in docker in different host servers on docker
first server 10.33.8.113
second server 10.33.8.79
first server elasticsearch yaml and docker-compose file
cat elasticsearch_node1.yml
cluster.name: “odfe-cluster”
node.name: odfe-node1
node.master: true
network.host: [site, local]
#network.publish_host: 0.0.0.0
transport.host: 0.0.0.0
transport.port: 9300
discovery.seed_hosts: “odfe-node1, odfe-node2”
cluster.initial_master_nodes: “odfe-node1, odfe-node2”
bootstrap.memory_lock: true
# minimum_master_nodes need to be explicitly set when bound on a public IP
# set to 1 to allow single node clusters
#discovery.zen.minimum_master_nodes: 2
logger.org.elasticsearch.discovery: TRACE
logger.org.elasticsearch.transport: TRACE
# Breaking change in 7.0
cluster.initial_master_nodes:
- elasticsearch1
- docker-test-node-1
######## Start OpenDistro for Elasticsearch Security Demo Configuration ########
WARNING: revise all the lines below before you go into production
opendistro_security.ssl.transport.pemcert_filepath: odfe-node1.pem
opendistro_security.ssl.transport.pemkey_filepath: odfe-node1.key
opendistro_security.ssl.transport.pemtrustedcas_filepath: MyRootCA.pem
opendistro_security.ssl.transport.enforce_hostname_verification: false
opendistro_security.ssl.transport.enabled_ciphers: “TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA”
opendistro_security.ssl.http.enabled: true
opendistro_security.ssl.http.pemcert_filepath: odfe-node1.pem
opendistro_security.ssl.http.pemkey_filepath: odfe-node1.key
opendistro_security.ssl.http.pemtrustedcas_filepath: MyRootCA.pem
opendistro_security.ssl.http.enabled_protocols:
- ‘TLSv1.2’
opendistro_security.nodes_dn:
- ‘CN=odfe-node1,OU=IT,O=ExampleOrg,L=SL,ST=SS,C=US’
- ‘CN=odfe-node2,OU=IT,O=ExampleOrg,L=SL,ST=SS,C=US’
- ‘CN=odfe-cluster*’
- ‘/CN=.*regex/’
#opendistro_security.allow_unsafe_democertificates: true ## Reoved for Selfigned cert
opendistro_security.allow_default_init_securityindex: true
opendistro_security.authcz.admin_dn:
- ‘CN=ADMIN,OU=IT,O=ExampleOrg,L=SL,ST=SS,C=US’
opendistro_security.audit.type: internal_elasticsearch
opendistro_security.enable_snapshot_restore_privilege: true
opendistro_security.check_snapshot_restore_write_privileges: true
opendistro_security.restapi.roles_enabled: [“all_access”, “security_rest_api_access”]
cluster.routing.allocation.disk.threshold_enabled: false
node.max_local_storage_nodes: 3
######## End OpenDistro for Elasticsearch Security Demo Configuration ########
azradmin@CI-STG-AIMEW-01:~/opendistro/elastic$
first server docker-compose file
cat docker-compose.yml
version: ‘3’
services:
odfe-node1:
image: amazon/opendistro-for-elasticsearch:1.13.3
restart: always
container_name: odfe-node1
environment:
- cluster.name=odfe-cluster
- node.name=odfe-node1
- network.host=0.0.0.0
- discovery.seed_hosts=odfe-node1, odfe-node2
- cluster.initial_master_nodes=odfe-node1,odfe-node2
- bootstrap.memory_lock=true # along with the memlock settings below, disables swapping
- “ES_JAVA_OPTS=-Xms6g -Xmx6g -Dhttps.protocols=TLSv1.2” # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 65536 # maximum number of open files for the Elasticsearch user, set to at least 65536 on modern systems
hard: 65536
volumes:
- odfe-data1:/usr/share/elasticsearch/data
- ./cert/MyRootCA.pem:/usr/share/elasticsearch/config/MyRootCA.pem
- ./cert/odfe-node1.pem:/usr/share/elasticsearch/config/odfe-node1.pem
- ./cert/odfe-node1.key:/usr/share/elasticsearch/config/odfe-node1.key
- ./cert/admin.pem:/usr/share/elasticsearch/config/admin.pem
- ./cert/admin-key.pem:/usr/share/elasticsearch/config/admin-key.pem
- ./elastic/elasticsearch_node1.yml:/usr/share/elasticsearch/config/elasticsearch.yml
- ./elastic/config/roles.yml:/usr/share/elasticsearch/plugins/opendistro_security/securityconfig/roles.yml
- ./elastic/config/roles_mapping.yml:/usr/share/elasticsearch/plugins/opendistro_security/securityconfig/roles_mapping.yml
- ./elastic/config/internal_users.yml:/usr/share/elasticsearch/plugins/opendistro_security/securityconfig/internal_users.yml
ports:
- 9200:9200
- 9600:9600 # required for Performance Analyzer
logging:
driver: json-file
options:
max-size: “100m”
max-file: “10”
networks:
- odfe-net
kibana:
image: amazon/opendistro-for-elasticsearch-kibana:1.13.2
container_name: odfe-kibana
ports:
- 5601:5601
expose:
- “5601”
environment:
ELASTICSEARCH_HOSTS: https://10.33.8.55:443 # keepalived virtual IP address using loadbalancer nginx
volumes:
- ./kibana/kibana.yml:/usr/share/kibana/config/kibana.yml
- ./cert/BBS_SRV_WEB_APP_SSL.crt:/usr/share/kibana/config/BBS_SRV_WEB_APP_SSL.crt
- ./cert/BBS_SRV_WEB_APP_SSL.key:/usr/share/kibana/config/BBS_SRV_WEB_APP_SSL.key
- ./cert/MyRootCA.pem:/usr/share/kibana/config/MyRootCA-.pem
- ./cert/AAI_BBS_Enterprise_Root_CA.crt:/usr/share/kibana/config/AAI_BBS_Enterprise_Root_CA.crt
networks:
- odfe-net
logging:
driver: json-file
options:
max-size: “100m”
max-file: “10”
logstash:
image: opensearchproject/logstash-oss-with-opensearch-output-plugin:7.16.2
container_name: logstash
volumes:
- ./logstash/event.conf:/usr/share/logstash/pipeline/event.conf
- ./logstash/CassandraJDBC42.jar:/usr/share/logstash/logstash-core/lib/jars/CassandraJDBC42.jar
- ./logstash/mysql.conf:/usr/share/logstash/pipeline/mysql.conf
- ./logstash/mysql-connector-java-8.0.22.jar:/usr/share/logstash/logstash-core/lib/jars/mysql-connector-java-8.0.22.jar
- ./cert/MyRootCA.pem:/usr/share/logstash/MyRootCA.pem
- ./logstash/jvm.options:/usr/share/logstash/config/jvm.options
ports:
- “5011:5011”
- “5012:5012”
- “5013:5013”
logging:
driver: json-file
options:
max-size: “512m”
max-file: “10”
networks:
- odfe-net
depends_on:
- odfe-node1
volumes:
odfe-data1:
#odfe-data2:
networks:
odfe-net:
second server elasticsearch.yaml
cluster.name: “odfe-cluster”
node.name: odfe-node2
node.master: true
network.host: [site, local]
#network.publish_host: 0.0.0.0
transport.host: 0.0.0.0
transport.port: 9300
discovery.seed_hosts: “odfe-node1, odfe-node2”
cluster.initial_master_nodes: “odfe-node1, odfe-node2”
bootstrap.memory_lock: true
# minimum_master_nodes need to be explicitly set when bound on a public IP
# set to 1 to allow single node clusters
#discovery.zen.minimum_master_nodes: 2
logger.org.elasticsearch.discovery: TRACE
logger.org.elasticsearch.transport: TRACE
# Breaking change in 7.0
cluster.initial_master_nodes:
- elasticsearch1
- docker-test-node-1
######## Start OpenDistro for Elasticsearch Security Demo Configuration ########
WARNING: revise all the lines below before you go into production
opendistro_security.ssl.transport.pemcert_filepath: odfe-node2.pem
opendistro_security.ssl.transport.pemkey_filepath: odfe-node2.key
opendistro_security.ssl.transport.pemtrustedcas_filepath: MyRootCA.pem
opendistro_security.ssl.transport.enforce_hostname_verification: false
opendistro_security.ssl.transport.enabled_ciphers: “TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA”
opendistro_security.ssl.http.enabled: true
opendistro_security.ssl.http.pemcert_filepath: odfe-node2.pem
opendistro_security.ssl.http.pemkey_filepath: odfe-node2.key
opendistro_security.ssl.http.pemtrustedcas_filepath: MyRootCA.pem
opendistro_security.ssl.http.enabled_protocols:
- ‘TLSv1.2’
opendistro_security.nodes_dn:
- ‘CN=odfe-node1,OU=IT,O=ExampleOrg,L=SL,ST=SS,C=US’
- ‘CN=odfe-node2,OU=IT,O=ExampleOrg,L=SL,ST=SS,C=US’
- ‘CN=odfe-cluster*’
- ‘/CN=.*regex/’
#opendistro_security.allow_unsafe_democertificates: true ## Reoved for Selfigned cert
opendistro_security.allow_default_init_securityindex: true
opendistro_security.authcz.admin_dn:
- ‘CN=ADMIN,OU=IT,O=ExampleOrg,L=SL,ST=SS,C=US’
opendistro_security.audit.type: internal_elasticsearch
opendistro_security.enable_snapshot_restore_privilege: true
opendistro_security.check_snapshot_restore_write_privileges: true
opendistro_security.restapi.roles_enabled: [“all_access”, “security_rest_api_access”]
cluster.routing.allocation.disk.threshold_enabled: false
node.max_local_storage_nodes: 3
End OpenDistro for Elasticsearch Security Demo Configuration
second server docker-compose file
cat docker-compose.yml
version: ‘3’
services:
odfe-node2:
image: amazon/opendistro-for-elasticsearch:1.13.3
restart: always
container_name: odfe-node2
environment:
- cluster.name=odfe-cluster
- node.name=odfe-node2
- network.host=0.0.0.0
#- transport.host=0.0.0.0
- discovery.seed_hosts=odfe-node1, odfe-node2
- cluster.initial_master_nodes=odfe-node1,odfe-node2
- bootstrap.memory_lock=true # along with the memlock settings below, disables swapping
- “ES_JAVA_OPTS=-Xms512m -Xmx512m -Dhttps.protocols=TLSv1.2” # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 65536 # maximum number of open files for the Elasticsearch user, set to at least 65536 on modern systems
hard: 65536
volumes:
- odfe-data2:/usr/share/elasticsearch/data
- ./cert/MyRootCA.pem:/usr/share/elasticsearch/config/MyRootCA.pem
- ./cert/odfe-node2.pem:/usr/share/elasticsearch/config/odfe-node2.pem
- ./cert/odfe-node2.key:/usr/share/elasticsearch/config/odfe-node2.key
- ./cert/admin.pem:/usr/share/elasticsearch/config/admin.pem
- ./cert/admin-key.pem:/usr/share/elasticsearch/config/admin-key.pem
- ./elastic/elasticsearch_node2.yml:/usr/share/elasticsearch/config/elasticsearch.yml
- ./elastic/config/roles.yml:/usr/share/elasticsearch/plugins/opendistro_security/securityconfig/roles.yml
- ./elastic/config/roles_mapping.yml:/usr/share/elasticsearch/plugins/opendistro_security/securityconfig/roles_mapping.yml
- ./elastic/config/internal_users.yml:/usr/share/elasticsearch/plugins/opendistro_security/securityconfig/internal_users.yml
ports:
- 9200:9200
- 9600:9600 # required for Performance Analyzer
logging:
driver: json-file
options:
max-size: “100m”
max-file: “10”
networks:
- odfe-net
volumes:
odfe-data2:
networks:
odfe-net:
Describe the issue:
both nodes getting same ERROE
i am getting peer connection failed we opend all ports in azure firewall and local firewall also still issue is same
LOGS
[2024-01-04T09:38:59,903][TRACE][o.e.d.HandshakingTransportAddressConnector] [odfe-node2] [connectToRemoteMasterNode[10.33.8.73:9300]] handshake successful: {odfe-node2}{PEVkwMFiSDejdmk4dcU0Gg}{r8qTL0V2TCG_477_OkdUzw}{172.22.0.2}{172.22.0.2:9300}{dimr}
[2024-01-04T09:38:59,903][DEBUG][o.e.t.TcpTransport ] [odfe-node2] closed transport connection [55] to [{10.33.8.73:9300}{WWHfWRdOTZuQk0OgA8So8A}{10.33.8.73}{10.33.8.73:9300}] with age [0ms]
[2024-01-04T09:38:59,903][DEBUG][o.e.d.PeerFinder ] [odfe-node2] Peer{transportAddress=10.33.8.73:9300, discoveryNode=null, peersRequestInFlight=false} connection failed
org.elasticsearch.transport.ConnectTransportException: [odfe-node2][172.22.0.2:9300] local node found
at org.elasticsearch.discovery.HandshakingTransportAddressConnector$1$1$1.innerOnResponse(HandshakingTransportAddressConnector.java:105) [elasticsearch-7.10.2.jar:7.10.2]
at org.elasticsearch.discovery.HandshakingTransportAddressConnector$1$1$1.innerOnResponse(HandshakingTransportAddressConnector.java:95) [elasticsearch-7.10.2.jar:7.10.2]
at org.elasticsearch.action.NotifyOnceListener.onResponse(NotifyOnceListener.java:40) [elasticsearch-7.10.2.jar:7.10.2]
at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:163) [elasticsearch-7.10.2.jar:7.10.2]
at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]
at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]
at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]
at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]
at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]
at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]
at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
at java.lang.Thread.run(Thread.java:832) [?:?]
[2024-01-04T09:38:59,905][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0xe90c573a, L:/172.22.0.2:37316 - R:/10.33.8.73:9300] CLOSE
[2024-01-04T09:38:59,906][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0xe90c573a, L:/172.22.0.2:37316 ! R:/10.33.8.73:9300] USER_EVENT: SslCloseCompletionEvent(java.nio.channels.ClosedChannelException)
[2024-01-04T09:38:59,906][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0xe90c573a, L:/172.22.0.2:37316 ! R:/10.33.8.73:9300] INACTIVE
[2024-01-04T09:38:59,906][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0x4ac72a6a, L:/172.22.0.2:9300 - R:/172.22.0.1:59830] USER_EVENT: SslCloseCompletionEvent(SUCCESS)
[2024-01-04T09:38:59,907][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0xe90c573a, L:/172.22.0.2:37316 ! R:/10.33.8.73:9300] UNREGISTERED
[2024-01-04T09:38:59,908][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0x4ac72a6a, L:/172.22.0.2:9300 ! R:/172.22.0.1:59830] INACTIVE
[2024-01-04T09:38:59,908][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0x4ac72a6a, L:/172.22.0.2:9300 ! R:/172.22.0.1:59830] UNREGISTERED
[2024-01-04T09:39:00,868][TRACE][o.e.d.PeerFinder ] [odfe-node2] probing master nodes from cluster state: nodes:
{odfe-node2}{PEVkwMFiSDejdmk4dcU0Gg}{r8qTL0V2TCG_477_OkdUzw}{172.22.0.2}{172.22.0.2:9300}{dimr}, local
[2024-01-04T09:39:00,868][TRACE][o.e.d.PeerFinder ] [odfe-node2] startProbe(172.22.0.2:9300) not probing local node
[2024-01-04T09:39:00,869][TRACE][o.e.d.SeedHostsResolver ] [odfe-node2] resolved host [10.33.8.134:9300] to [10.33.8.134:9300]
[2024-01-04T09:39:00,869][TRACE][o.e.d.SeedHostsResolver ] [odfe-node2] resolved host [10.33.8.73:9300] to [10.33.8.73:9300]
[2024-01-04T09:39:00,870][TRACE][o.e.d.PeerFinder ] [odfe-node2] probing resolved transport addresses [10.33.8.134:9300, 10.33.8.73:9300]
[2024-01-04T09:39:00,870][TRACE][o.e.d.PeerFinder ] [odfe-node2] Peer{transportAddress=10.33.8.73:9300, discoveryNode=null, peersRequestInFlight=false} attempting connection
[2024-01-04T09:39:00,870][TRACE][o.e.d.HandshakingTransportAddressConnector] [odfe-node2] [connectToRemoteMasterNode[10.33.8.73:9300]] opening probe connection
[2024-01-04T09:39:00,871][TRACE][o.e.t.TcpTransport ] [odfe-node2] Tcp transport client channel opened: Netty4TcpChannel{localAddress=null, remoteAddress=null}
[2024-01-04T09:39:00,872][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0xa6be2401] REGISTERED
[2024-01-04T09:39:00,872][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0xa6be2401] CONNECT: /10.33.8.73:9300
[2024-01-04T09:39:00,873][TRACE][o.e.t.TcpTransport ] [odfe-node2] Tcp transport channel accepted: Netty4TcpChannel{localAddress=/172.22.0.2:9300, remoteAddress=/172.22.0.1:59840}
[2024-01-04T09:39:00,874][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0x1fa62e8d, L:/172.22.0.2:9300 - R:/172.22.0.1:59840] REGISTERED
[2024-01-04T09:39:00,874][TRACE][o.e.t.TransportLogger ] [odfe-node2] Netty4TcpChannel{localAddress=/172.22.0.2:37318, remoteAddress=/10.33.8.73:9300} [length: 51, request id: 111, type: request, version: 6.8.0, action: internal:tcp/handshake] WRITE: 51B
[2024-01-04T09:39:00,875][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0x1fa62e8d, L:/172.22.0.2:9300 - R:/172.22.0.1:59840] ACTIVE
[2024-01-04T09:39:00,875][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0xa6be2401, L:/172.22.0.2:37318 - R:/10.33.8.73:9300] WRITE: 51B
±------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
±-------±------------------------------------------------±---------------+
|00000000| 45 53 00 00 00 2d 00 00 00 00 00 00 00 6f 08 00 |ES…-…o…|
|00000010| 5c c6 63 00 00 00 16 69 6e 74 65 72 6e 61 6c 3a |.c…internal:|
|00000020| 74 63 70 2f 68 61 6e 64 73 68 61 6b 65 00 04 8b |tcp/handshake…|
|00000030| af b1 03 |… |
±-------±------------------------------------------------±---------------+
[2024-01-04T09:39:00,875][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0xa6be2401, L:/172.22.0.2:37318 - R:/10.33.8.73:9300] FLUSH
[2024-01-04T09:39:00,877][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0xa6be2401, L:/172.22.0.2:37318 - R:/10.33.8.73:9300] ACTIVE
[2024-01-04T09:39:00,891][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0x1fa62e8d, L:/172.22.0.2:9300 - R:/172.22.0.1:59840] USER_EVENT: SslHandshakeCompletionEvent(SUCCESS)
[2024-01-04T09:39:00,892][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0xa6be2401, L:/172.22.0.2:37318 - R:/10.33.8.73:9300] USER_EVENT: SslHandshakeCompletionEvent(SUCCESS)
[2024-01-04T09:39:00,893][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0x1fa62e8d, L:/172.22.0.2:9300 - R:/172.22.0.1:59840] READ: 51B
±------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
±-------±------------------------------------------------±---------------+
|00000000| 45 53 00 00 00 2d 00 00 00 00 00 00 00 6f 08 00 |ES…-…o…|
|00000010| 5c c6 63 00 00 00 16 69 6e 74 65 72 6e 61 6c 3a |.c…internal:|
|00000020| 74 63 70 2f 68 61 6e 64 73 68 61 6b 65 00 04 8b |tcp/handshake…|
|00000030| af b1 03 |… |
±-------±------------------------------------------------±---------------+
[2024-01-04T09:39:00,893][TRACE][o.e.t.TransportLogger ] [odfe-node2] Netty4TcpChannel{localAddress=/172.22.0.2:9300, remoteAddress=/172.22.0.1:59840} [length: 51, request id: 111, type: request, version: 6.8.0, action: internal:tcp/handshake] READ: 51B
[2024-01-04T09:39:00,893][TRACE][o.e.t.T.tracer ] [odfe-node2] [111][internal:tcp/handshake] received request
[2024-01-04T09:39:00,893][TRACE][o.e.t.TransportLogger ] [odfe-node2] Netty4TcpChannel{localAddress=/172.22.0.2:9300, remoteAddress=/172.22.0.1:59840} [length: 25, request id: 111, type: response, version: 6.8.0] WRITE: 25B
[2024-01-04T09:39:00,893][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0x1fa62e8d, L:/172.22.0.2:9300 - R:/172.22.0.1:59840] WRITE: 25B
±------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
±-------±------------------------------------------------±---------------+
|00000000| 45 53 00 00 00 13 00 00 00 00 00 00 00 6f 09 00 |ES…o…|
|00000010| 5c c6 63 00 00 8b af b1 03 |.c… |
±-------±------------------------------------------------±---------------+
[2024-01-04T09:39:00,893][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0x1fa62e8d, L:/172.22.0.2:9300 - R:/172.22.0.1:59840] FLUSH
[2024-01-04T09:39:00,894][TRACE][o.e.t.T.tracer ] [odfe-node2] [111][internal:tcp/handshake] sent to [{10.33.8.73:9300}{gnobVIf7TPaqUsJr1bKgHQ}{10.33.8.73}{10.33.8.73:9300}] (timeout: [null])
[2024-01-04T09:39:00,894][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0xa6be2401, L:/172.22.0.2:37318 - R:/10.33.8.73:9300] READ: 25B
±------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
±-------±------------------------------------------------±---------------+
|00000000| 45 53 00 00 00 13 00 00 00 00 00 00 00 6f 09 00 |ES…o…|
|00000010| 5c c6 63 00 00 8b af b1 03 |.c… |
±-------±------------------------------------------------±---------------+
[2024-01-04T09:39:00,894][TRACE][o.e.t.T.tracer ] [odfe-node2] [111][internal:tcp/handshake] sent response
[2024-01-04T09:39:00,894][TRACE][o.e.t.TransportLogger ] [odfe-node2] Netty4TcpChannel{localAddress=/172.22.0.2:37318, remoteAddress=/10.33.8.73:9300} [length: 25, request id: 111, type: response, version: 6.8.0] READ: 25B
[2024-01-04T09:39:00,895][DEBUG][o.e.t.TcpTransport ] [odfe-node2] opened transport connection [56] to [{10.33.8.73:9300}{gnobVIf7TPaqUsJr1bKgHQ}{10.33.8.73}{10.33.8.73:9300}] using channels [[Netty4TcpChannel{localAddress=/172.22.0.2:37318, remoteAddress=/10.33.8.73:9300}]]
[2024-01-04T09:39:00,895][TRACE][o.e.d.HandshakingTransportAddressConnector] [odfe-node2] [connectToRemoteMasterNode[10.33.8.73:9300]] opened probe connection
[2024-01-04T09:39:00,896][TRACE][o.e.t.TransportLogger ] [odfe-node2] Netty4TcpChannel{localAddress=/172.22.0.2:37318, remoteAddress=/10.33.8.73:9300} [length: 140, request id: 112, type: request, version: 7.10.2, header size: 116B, action: internal:transport/handshake] WRITE: 140B
[2024-01-04T09:39:00,896][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0xa6be2401, L:/172.22.0.2:37318 - R:/10.33.8.73:9300] WRITE: 140B
±------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
±-------±------------------------------------------------±---------------+
|00000000| 45 53 00 00 00 86 00 00 00 00 00 00 00 70 00 00 |ES…p…|
|00000010| 6c 57 8b 00 00 00 74 02 1d 5f 6f 70 65 6e 64 69 |lW…t…opendi|
|00000020| 73 74 72 6f 5f 73 65 63 75 72 69 74 79 5f 72 65 |stro_security_re|
|00000030| 6d 6f 74 65 63 6e 0c 6f 64 66 65 2d 63 6c 75 73 |motecn.odfe-clus|
|00000040| 74 65 72 22 5f 6f 70 65 6e 64 69 73 74 72 6f 5f |ter"opendistro|
|00000050| 73 65 63 75 72 69 74 79 5f 6f 72 69 67 69 6e 5f |security_origin|
|00000060| 68 65 61 64 65 72 05 4c 4f 43 41 4c 00 00 1c 69 |header.LOCAL…i|
|00000070| 6e 74 65 72 6e 61 6c 3a 74 72 61 6e 73 70 6f 72 |nternal:transpor|
|00000080| 74 2f 68 61 6e 64 73 68 61 6b 65 00 |t/handshake. |
±-------±------------------------------------------------±---------------+
[2024-01-04T09:39:00,896][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0xa6be2401, L:/172.22.0.2:37318 - R:/10.33.8.73:9300] FLUSH
[2024-01-04T09:39:00,897][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0x1fa62e8d, L:/172.22.0.2:9300 - R:/172.22.0.1:59840] READ: 140B
±------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
±-------±------------------------------------------------±---------------+
|00000000| 45 53 00 00 00 86 00 00 00 00 00 00 00 70 00 00 |ES…p…|
|00000010| 6c 57 8b 00 00 00 74 02 1d 5f 6f 70 65 6e 64 69 |lW…t…opendi|
|00000020| 73 74 72 6f 5f 73 65 63 75 72 69 74 79 5f 72 65 |stro_security_re|
|00000030| 6d 6f 74 65 63 6e 0c 6f 64 66 65 2d 63 6c 75 73 |motecn.odfe-clus|
|00000040| 74 65 72 22 5f 6f 70 65 6e 64 69 73 74 72 6f 5f |ter"opendistro|
|00000050| 73 65 63 75 72 69 74 79 5f 6f 72 69 67 69 6e 5f |security_origin|
|00000060| 68 65 61 64 65 72 05 4c 4f 43 41 4c 00 00 1c 69 |header.LOCAL…i|
|00000070| 6e 74 65 72 6e 61 6c 3a 74 72 61 6e 73 70 6f 72 |nternal:transpor|
|00000080| 74 2f 68 61 6e 64 73 68 61 6b 65 00 |t/handshake. |
±-------±------------------------------------------------±---------------+
[2024-01-04T09:39:00,897][TRACE][o.e.t.TransportLogger ] [odfe-node2] Netty4TcpChannel{localAddress=/172.22.0.2:9300, remoteAddress=/172.22.0.1:59840} [length: 140, request id: 112, type: request, version: 7.10.2, action: internal:transport/handshake] READ: 140B
[2024-01-04T09:39:00,897][TRACE][o.e.t.T.tracer ] [odfe-node2] [112][internal:transport/handshake] received request
[2024-01-04T09:39:00,898][TRACE][o.e.t.T.tracer ] [odfe-node2] [112][internal:transport/handshake] sent to [{10.33.8.73:9300}{gnobVIf7TPaqUsJr1bKgHQ}{10.33.8.73}{10.33.8.73:9300}] (timeout: [1s])
[2024-01-04T09:39:00,898][TRACE][o.e.t.TransportLogger ] [odfe-node2] Netty4TcpChannel{localAddress=/172.22.0.2:9300, remoteAddress=/172.22.0.1:59840} [length: 351, request id: 112, type: response, version: 7.10.2, header size: 152B] WRITE: 351B
[2024-01-04T09:39:00,899][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0x1fa62e8d, L:/172.22.0.2:9300 - R:/172.22.0.1:59840] WRITE: 351B
±------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
±-------±------------------------------------------------±---------------+
|00000000| 45 53 00 00 01 59 00 00 00 00 00 00 00 70 01 00 |ES…Y…p…|
|00000010| 6c 57 8b 00 00 00 98 03 1d 5f 6f 70 65 6e 64 69 |lW…opendi|
|00000020| 73 74 72 6f 5f 73 65 63 75 72 69 74 79 5f 72 65 |stro_security_re|
|00000030| 6d 6f 74 65 63 6e 0c 6f 64 66 65 2d 63 6c 75 73 |motecn.odfe-clus|
|00000040| 74 65 72 30 5f 6f 70 65 6e 64 69 73 74 72 6f 5f |ter0_opendistro|
|00000050| 73 65 63 75 72 69 74 79 5f 69 6e 69 74 69 61 6c |security_initial|
|00000060| 5f 61 63 74 69 6f 6e 5f 63 6c 61 73 73 5f 68 65 |_action_class_he|
|00000070| 61 64 65 72 10 48 61 6e 64 73 68 61 6b 65 52 65 |ader.HandshakeRe|
|00000080| 71 75 65 73 74 22 5f 6f 70 65 6e 64 69 73 74 72 |quest"_opendistr|
|00000090| 6f 5f 73 65 63 75 72 69 74 79 5f 6f 72 69 67 69 |o_security_origi|
|000000a0| 6e 5f 68 65 61 64 65 72 05 4c 4f 43 41 4c 00 01 |n_header.LOCAL…|
|000000b0| 0a 6f 64 66 65 2d 6e 6f 64 65 32 16 50 45 56 6b |.odfe-node2.PEVk|
|000000c0| 77 4d 46 69 53 44 65 6a 64 6d 6b 34 64 63 55 30 |wMFiSDejdmk4dcU0|
|000000d0| 47 67 16 72 38 71 54 4c 30 56 32 54 43 47 5f 34 |Gg.r8qTL0V2TCG_4|
|000000e0| 37 37 5f 4f 6b 64 55 7a 77 0a 31 37 32 2e 32 32 |77_OkdUzw.172.22|
|000000f0| 2e 30 2e 32 0a 31 37 32 2e 32 32 2e 30 2e 32 04 |.0.2.172.22.0.2.|
|00000100| ac 16 00 02 0a 31 37 32 2e 32 32 2e 30 2e 32 00 |…172.22.0.2.|
|00000110| 00 24 54 00 04 04 64 61 74 61 01 64 01 06 69 6e |.$T…data.d…in|
|00000120| 67 65 73 74 01 69 00 06 6d 61 73 74 65 72 01 6d |gest.i…master.m|
|00000130| 00 15 72 65 6d 6f 74 65 5f 63 6c 75 73 74 65 72 |…remote_cluster|
|00000140| 5f 63 6c 69 65 6e 74 01 72 00 8b af b1 03 0c 6f |_client.r…o|
|00000150| 64 66 65 2d 63 6c 75 73 74 65 72 8b af b1 03 |dfe-cluster… |
±-------±------------------------------------------------±---------------+
[2024-01-04T09:39:00,899][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0x1fa62e8d, L:/172.22.0.2:9300 - R:/172.22.0.1:59840] FLUSH
[2024-01-04T09:39:00,900][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0xa6be2401, L:/172.22.0.2:37318 - R:/10.33.8.73:9300] READ: 351B
±------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
±-------±------------------------------------------------±---------------+
|00000000| 45 53 00 00 01 59 00 00 00 00 00 00 00 70 01 00 |ES…Y…p…|
|00000010| 6c 57 8b 00 00 00 98 03 1d 5f 6f 70 65 6e 64 69 |lW…opendi|
|00000020| 73 74 72 6f 5f 73 65 63 75 72 69 74 79 5f 72 65 |stro_security_re|
|00000030| 6d 6f 74 65 63 6e 0c 6f 64 66 65 2d 63 6c 75 73 |motecn.odfe-clus|
|00000040| 74 65 72 30 5f 6f 70 65 6e 64 69 73 74 72 6f 5f |ter0_opendistro|
|00000050| 73 65 63 75 72 69 74 79 5f 69 6e 69 74 69 61 6c |security_initial|
|00000060| 5f 61 63 74 69 6f 6e 5f 63 6c 61 73 73 5f 68 65 |_action_class_he|
|00000070| 61 64 65 72 10 48 61 6e 64 73 68 61 6b 65 52 65 |ader.HandshakeRe|
|00000080| 71 75 65 73 74 22 5f 6f 70 65 6e 64 69 73 74 72 |quest"_opendistr|
|00000090| 6f 5f 73 65 63 75 72 69 74 79 5f 6f 72 69 67 69 |o_security_origi|
|000000a0| 6e 5f 68 65 61 64 65 72 05 4c 4f 43 41 4c 00 01 |n_header.LOCAL…|
|000000b0| 0a 6f 64 66 65 2d 6e 6f 64 65 32 16 50 45 56 6b |.odfe-node2.PEVk|
|000000c0| 77 4d 46 69 53 44 65 6a 64 6d 6b 34 64 63 55 30 |wMFiSDejdmk4dcU0|
|000000d0| 47 67 16 72 38 71 54 4c 30 56 32 54 43 47 5f 34 |Gg.r8qTL0V2TCG_4|
|000000e0| 37 37 5f 4f 6b 64 55 7a 77 0a 31 37 32 2e 32 32 |77_OkdUzw.172.22|
|000000f0| 2e 30 2e 32 0a 31 37 32 2e 32 32 2e 30 2e 32 04 |.0.2.172.22.0.2.|
|00000100| ac 16 00 02 0a 31 37 32 2e 32 32 2e 30 2e 32 00 |…172.22.0.2.|
|00000110| 00 24 54 00 04 04 64 61 74 61 01 64 01 06 69 6e |.$T…data.d…in|
|00000120| 67 65 73 74 01 69 00 06 6d 61 73 74 65 72 01 6d |gest.i…master.m|
|00000130| 00 15 72 65 6d 6f 74 65 5f 63 6c 75 73 74 65 72 |…remote_cluster|
|00000140| 5f 63 6c 69 65 6e 74 01 72 00 8b af b1 03 0c 6f |_client.r…o|
|00000150| 64 66 65 2d 63 6c 75 73 74 65 72 8b af b1 03 |dfe-cluster… |
±-------±------------------------------------------------±---------------+
[2024-01-04T09:39:00,900][TRACE][o.e.t.TransportLogger ] [odfe-node2] Netty4TcpChannel{localAddress=/172.22.0.2:37318, remoteAddress=/10.33.8.73:9300} [length: 351, request id: 112, type: response, version: 7.10.2] READ: 351B
[2024-01-04T09:39:00,900][TRACE][o.e.t.T.tracer ] [odfe-node2] [112][internal:transport/handshake] received response from [{10.33.8.73:9300}{gnobVIf7TPaqUsJr1bKgHQ}{10.33.8.73}{10.33.8.73:9300}]
[2024-01-04T09:39:00,904][TRACE][o.e.t.T.tracer ] [odfe-node2] [112][internal:transport/handshake] sent response
[2024-01-04T09:39:00,904][TRACE][o.e.d.HandshakingTransportAddressConnector] [odfe-node2] [connectToRemoteMasterNode[10.33.8.73:9300]] handshake successful: {odfe-node2}{PEVkwMFiSDejdmk4dcU0Gg}{r8qTL0V2TCG_477_OkdUzw}{172.22.0.2}{172.22.0.2:9300}{dimr}
[2024-01-04T09:39:00,904][DEBUG][o.e.t.TcpTransport ] [odfe-node2] closed transport connection [56] to [{10.33.8.73:9300}{gnobVIf7TPaqUsJr1bKgHQ}{10.33.8.73}{10.33.8.73:9300}] with age [0ms]
[2024-01-04T09:39:00,904][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0xa6be2401, L:/172.22.0.2:37318 - R:/10.33.8.73:9300] CLOSE
[2024-01-04T09:39:00,904][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0x1fa62e8d, L:/172.22.0.2:9300 - R:/172.22.0.1:59840] USER_EVENT: SslCloseCompletionEvent(SUCCESS)
[2024-01-04T09:39:00,905][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0xa6be2401, L:/172.22.0.2:37318 ! R:/10.33.8.73:9300] USER_EVENT: SslCloseCompletionEvent(java.nio.channels.ClosedChannelException)
[2024-01-04T09:39:00,905][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0xa6be2401, L:/172.22.0.2:37318 ! R:/10.33.8.73:9300] INACTIVE
[2024-01-04T09:39:00,905][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0xa6be2401, L:/172.22.0.2:37318 ! R:/10.33.8.73:9300] UNREGISTERED
[2024-01-04T09:39:00,906][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0x1fa62e8d, L:/172.22.0.2:9300 ! R:/172.22.0.1:59840] INACTIVE
[2024-01-04T09:39:00,906][TRACE][o.e.t.n.ESLoggingHandler ] [odfe-node2] [id: 0x1fa62e8d, L:/172.22.0.2:9300 ! R:/172.22.0.1:59840] UNREGISTERED
[2024-01-04T09:39:00,904][DEBUG][o.e.d.PeerFinder ] [odfe-node2] Peer{transportAddress=10.33.8.73:9300, discoveryNode=null, peersRequestInFlight=false} connection failed
org.elasticsearch.transport.ConnectTransportException: [odfe-node2][172.22.0.2:9300] local node found
at org.elasticsearch.discovery.HandshakingTransportAddressConnector$1$1$1.innerOnResponse(HandshakingTransportAddressConnector.java:105) [elasticsearch-7.10.2.jar:7.10.2]
at org.elasticsearch.discovery.HandshakingTransportAddressConnector$1$1$1.innerOnResponse(HandshakingTransportAddressConnector.java:95) [elasticsearch-7.10.2.jar:7.10.2]
at org.elasticsearch.action.NotifyOnceListener.onResponse(NotifyOnceListener.java:40) [elasticsearch-7.10.2.jar:7.10.2]
at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:163) [elasticsearch-7.10.2.jar:7.10.2]
at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]
at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]
at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]
at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]
at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]
at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]
at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
at java.lang.Thread.run(Thread.java:832) [?:?]
Configuration:
Relevant Logs or Screenshots:
please help me in this error
- when giving network.host : <vm’s IP> i am getting this error
[2024-01-04T09:09:22,728][WARN ][o.e.d.HandshakingTransportAddressConnector] [odfe-node2] [connectToRemoteMasterNode[10.33.8.134:9300]] completed handshake with [{odfe-node1}{Qjpzkls_QGGujpmyslmmiQ}{ADdWigR0TOmsn4dJxTEPqA}{172.20.0.2}{172.20.0.2:9300}{dimr}] but followup connection failed
org.elasticsearch.transport.ConnectTransportException: [odfe-node1][172.20.0.2:9300] connect_exception
at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]
at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]
at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]
at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]
at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]
at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]
at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]
at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]
at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe$1.run(AbstractNioChannel.java:263) ~[?:?]
at io.netty.util.concurrent.PromiseTask.runTask(PromiseTask.java:98) ~[?:?]
at io.netty.util.concurrent.ScheduledFutureTask.run(ScheduledFutureTask.java:170) ~[?:?]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164) ~[?:?]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[?:?]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:497) ~[?:?]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]
at java.lang.Thread.run(Thread.java:832) [?:?]
Caused by: io.netty.channel.ConnectTimeoutException: connection timed out: 172.20.0.2/172.20.0.2:9300
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe$1.run(AbstractNioChannel.java:261) ~[?:?]
… 8 more