Hi all - new to the opensearch community and still excited to learn more…
curios, engaged, challenged, frustrated, dantes 5’th leve.. and before it goes further down hill, I’m raising the flag to ask for help.
v.2.19.1
Docker and RHEL
5 nodes (3 master)
Self signed x509 certs in .pem format
Cluster members report javax.ent.ssl.SSLHandshake Exception: InvalidCertificateVerify
Configuration:
***sample node docker-config.yml
services:
opensearch-ingest:
image: opensearchproject/opensearch:2.19.1
container_name: dev-os-ingest
environment:
- cluster.name=dev-os-shfl
- node.name=dev-os-ingest
- node.roles=[cluster_manager,ingest]
- discovery.seed_hosts=dev-os-ingest,dev-os-dash,dev-os-data,dev-os-ml,dev-os-data2
- cluster.initial_master_nodes=dev-os-ingest,dev-os-data,dev-os-data2
- bootstrap.memory_lock=true
- "DISABLE_INSTALL_DEMO_CONFIG=true"
- "DISABLE_SECURITY_PLUGIN=false" #Activate for Security
- "OPENSEARCH_JAVA_OPTS=-Xms2g -Xmx2g"
- "OPENSEARCH_INITIAL_ADMIN_PASSWORD=redact"
- network.host=_local_,_site_
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 65536
hard: 65536
volumes:
- opensearch-data1:/usr/share/opensearch/data
- ./config/opensearch.yml:/usr/share/opensearch/config/opensearch.yml
- ./config/root-ca.pem:/usr/share/opensearch/config/root-ca.pem
- ./config/dev-admin.pem:/usr/share/opensearch/config/dev-admin.pem
- ./config/admin-key.pem:/usr/share/opensearch/config/admin-key.pem
- ./config/dev-os-ingest.pem:/usr/share/opensearch/config/dev-os-ingest.pem
- ./config/dev-os-ingest-key.pem:/usr/share/opensearch/config/dev-os-ingest-key.pem
network_mode: host
extra_hosts:
- "dev-os-ingest dev-os-ingest.int.redact.com:10.xyz.10.132"
- "dev-os-dash dev-os-coord dev-os-dash.redact.com dev-os-coord.int.redact.com:10.xyz.10.133"
- "dev-os-data dev-os-data.int.redact.com:10.xyz.10.134"
- "dev-os-ml dev-os-ml.int.redact.com:10.xyz.10.135"
- "dev-os-data2 dev-os-data2.int.redact.com:10.xyz.10.137"
volumes:
opensearch-data1:
Sample opensearch.yml*
plugins.security.ssl.transport.pemcert_filepath: /usr/share/opensearch/config/dev-os-ingest.pem
plugins.security.ssl.transport.pemkey_filepath: /usr/share/opensearch/config/dev-os-ingest-key.pem
plugins.security.ssl.transport.pemtrustedcas_filepath: /usr/share/opensearch/config/root-ca.pem
plugins.security.ssl.http.pemcert_filepath: /usr/share/opensearch/config/dev-os-ingest.pem
plugins.security.ssl.http.pemkey_filepath: /usr/share/opensearch/config/dev-os-ingest-key.pem
plugins.security.ssl.http.pemtrustedcas_filepath: /usr/share/opensearch/config/root-ca.pem
plugins.security.allow_default_init_securityindex: true
plugins.security.ssl.transport.enabled: true
plugins.security.ssl.http.enabled: true
plugins.security.ssl.transport.enforce_hostname_verification: false
plugins.security.ssl.http.enabled_ciphers:
- "TLS_DHE_RSA_WITH_AES_256_CBC_SHA"
- "TLS_DHE_DSS_WITH_AES_128_CBC_SHA256"
plugins.security.ssl.http.enabled_protocols:
- "TLSv1.3"
- "TLSv1.2"
plugins.security.authcz.admin_dn:
- 'CN=A,OU=Dev,O=redact yx4,L=city,ST=State,C=US'
plugins.security.nodes_dn:
- 'CN=dev-os-data2.int.redact.com,OU=DEV,O=redact yx4,L=city,ST=State,C=US'
- 'CN=dev-os-data.int.redact.com,OU=DEV,O=redact yx4,L=city,ST=State,C=US'
- 'CN=dev-os-ml.int.redact.com,OU=DEV,O=redact yx4,L=city,ST=State,C=US'
- 'CN=dev-os-coord.int.redact.com,OU=DEV,O=redact yx4,L=city,ST=State,C=US'
- 'CN=dev-os-ingest.int.redact.com,OU=DEV,O=redact yx4,L=city,ST=State,C=US'
Relevant Logs or Screenshots:
- ca signed certs appear to be valid
[root@dev-os-ingest config]# openssl verify -verbose -CAfile root-ca.pem dev-os-ingest.pem
dev-os-ingest.pem: OK - verify ssl is valid
[root@dev-os-ingest config]# openssl s_client -connect dev-os-ingest:9200
Connecting to 10.254.10.132
CONNECTED(00000003)
SSL handshake has read 3560 bytes and written 656 bytes
Verification: OK
---
New, SSLv3, Cipher is DHE-RSA-AES256-SHA
Server public key is 2048 bit
Secure Renegotiation IS supported
Compression: NONE
Expansion: NONE
No ALPN negotiated
SSL-Session:
Protocol : TLSv1.2
Cipher : DHE-RSA-AES256-SHA
- Ran securityadmin.sh
sh-5.2$ ./securityadmin.sh -cacert /usr/share/opensearch/config/root-ca.pem -cert /usr/share/opensearch/config/dev-admin.pem -key /usr/share/opensearch/config/admin-key.pem -cd ../../../config/opensearch-security/
Security Admin v7
Will connect to localhost:9200 ... done
ERR: An unexpected SSLHandshakeException occured: Received fatal alert: handshake_failure
See https://opensearch.org/docs/latest/clients/java-rest-high-level/ for troubleshooting.
Trace:
javax.net.ssl.SSLHandshakeException: Received fatal alert: handshake_failure
See https://opensearch.org/docs/latest/clients/java-rest-high-level/ for troubleshooting.
at org.opensearch.client.RestClient.extractAndWrapCause(RestClient.java:1241)
at org.opensearch.client.RestClient.performRequest(RestClient.java:358)
at org.opensearch.client.RestClient.performRequest(RestClient.java:346)
at org.opensearch.security.tools.SecurityAdmin.execute(SecurityAdmin.java:575)
at org.opensearch.security.tools.SecurityAdmin.main(SecurityAdmin.java:165)
Caused by: javax.net.ssl.SSLHandshakeException: Received fatal alert: handshake_failure
at java.base/sun.security.ssl.Alert.createSSLException(Alert.java:130)
at java.base/sun.security.ssl.Alert.createSSLException(Alert.java:117)
at java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:370)
at java.base/sun.security.ssl.Alert$AlertConsumer.consume(Alert.java:287)
at java.base/sun.security.ssl.TransportContext.dispatch(TransportContext.java:209)
at java.base/sun.security.ssl.SSLTransport.decode(SSLTransport.java:172)
at java.base/sun.security.ssl.SSLEngineImpl.decode(SSLEngineImpl.java:736)
at java.base/sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:691)
at java.base/sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:506)
at java.base/sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:482)
at java.base/javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:679)
at org.apache.http.nio.reactor.ssl.SSLIOSession.doUnwrap(SSLIOSession.java:279)
at org.apache.http.nio.reactor.ssl.SSLIOSession.doHandshake(SSLIOSession.java:333)
at org.apache.http.nio.reactor.ssl.SSLIOSession.isAppInputReady(SSLIOSession.java:545)
at org.apache.http.impl.nio.reactor.AbstractIODispatch.inputReady(AbstractIODispatch.java:120)
at org.apache.http.impl.nio.reactor.BaseIOReactor.readable(BaseIOReactor.java:162)
at org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvent(AbstractIOReactor.java:337)
at org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvents(AbstractIOReactor.java:315)
at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:276)
at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104)
at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:591)
at java.base/java.lang.Thread.run(Thread.java:1583)
- Verified cert SANs and extensions
[root@dev-os-ingest config]# Certificate:
Data: openssl x509 -in dev-os-ingest.pem -text -noout
x509
Certificate:
Data:
Version: 3 (0x2)
Serial Number:
18:64:33:f5:f8xyz
Signature Algorithm: sha256WithRSAEncryption
Issuer: C=US, ST=NO, L=PDQ, O=redact, OU=dev, CN=int.redact.com
Validity
Not Before: Mar 19 16:00:56 2025 GMT
Not After : Mar 17 16:00:56 2035 GMT
Subject: C=US, ST=NO, L=PDQ, O=redact, OU=DEV, CN=dev-os-ingest.int.redact.com
Subject Public Key Info:
Public Key Algorithm: rsaEncryption
Public-Key: (2048 bit)
Modulus:
00:b2:4d:d8:00:f2:28:89:48:4b:36:13:b9:63:98:
b1:6e:80:3c:99:6d:de:15:dc:00:1d:87:de:cb:b1:
42:b9
Exponent: 65537 (0x10001)
X509v3 extensions:
X509v3 Key Usage:
Digital Signature, Key Encipherment, Data Encipherment
X509v3 Extended Key Usage:
TLS Web Server Authentication, TLS Web Client Authentication
X509v3 Subject Alternative Name:
IP Address:10.xuz.10.132, DNS:dev-os-ingest.int.redact.com, DNS:dev-os-ingest
X509v3 Subject Key Identifier:
72:A4:9B:
X509v3 Authority Key Identifier:
BF:87:24:
Signature Algorithm: sha256WithRSAEncryption
Signature Value:
6d:a8:10:7c:ba:9e:e0:24:fd:0a:5d:91:cb:e1:5a:ca:89:76:
ff:91:53:50:a8:68:c8:f0:68:e6:da:e7:55:a9:50:a7:8d:25:
48:60:7e:a6
Thanks in advance for sharing your experience and taking a look with a fresh set of eyes. Humbly optimistic that I just missed something simple.