Hi,
OpenSearch: v2.13
system: ubuntu 22.04
docker based installation
There are two nodes: manager1.opensearch.test.com and data1.opensearch.test.com.
And there is opensearch dashboard: dashboard.opensearch.test.com.
manager1, data1 and dashboard are in dockers and proxied by nginx.
On manager1 node opensearch in docker with cluster_manager role is running inside docker container.
On data1 node – opensearch with data,ingest roles.
I use self signed certificates for configuring TLS/SSL connection between these two nodes according to this page.
TLS communication is normally configured between manager1 and data1 nodes (cluster is in green state)
But when i try to configure TLS communication between manager1 and dashboard i get errors on dashobard side
[ConnectionError]: unable to get local issuer certificate
Question: What should i change for successful TLS connection between dashboard and manager1?
Commands for generating certificates:
openssl genpkey -out root-ca.key -algorithm RSA -pkeyopt rsa_keygen_bits:2048
openssl req -new -x509 -sha256 -key root-ca.key -subj "/C=GB/L=London/O=test/CN=opensearch.test.com" -out root-ca.pem -days 800
openssl genpkey -out admin/admin.key -algorithm RSA -pkeyopt rsa_keygen_bits:2048
openssl pkcs8 -inform PEM -outform PEM -in admin/admin.key -topk8 -nocrypt -v1 PBE-SHA1-3DES -out admin/admin_pkcs8.key
openssl req -new -key admin/admin_pkcs8.key -subj "/C=GB/L=London/O=test/CN=admin.opensearch.test.com" -out admin/admin.csr
openssl x509 -req -in admin/admin.csr -CA root-ca.pem -CAkey root-ca.key -sha256 -out admin/admin.pem -days 800
openssl genpkey -out manager1/manager1.key -algorithm RSA -pkeyopt rsa_keygen_bits:2048
openssl pkcs8 -inform PEM -outform PEM -in manager1/manager1.key -topk8 -nocrypt -v1 PBE-SHA1-3DES -out manager1/manager1_pkcs8.key
openssl req -new -key manager1/manager1_pkcs8.key -subj "/C=GB/L=London/O=test/CN=manager1.opensearch.test.com" -out manager1/manager1.csr
echo 'subjectAltName=DNS:manager1.opensearch.test.com' > manager1/manager1.ext
openssl x509 -req -in manager1/manager1.csr -CA root-ca.pem -CAkey root-ca.key -sha256 -out manager1/manager1.pem -days 800 -extfile manager1/manager1.ext
openssl genpkey -out data1/data1.key -algorithm RSA -pkeyopt rsa_keygen_bits:2048
openssl pkcs8 -inform PEM -outform PEM -in data1/data1.key -topk8 -nocrypt -v1 PBE-SHA1-3DES -out data1/data1_pkcs8.key
openssl req -new -key data1/data1_pkcs8.key -subj "/C=GB/L=London/O=test/CN=data1.opensearch.test.com" -out data1/data1.csr
echo 'subjectAltName=DNS:data1.opensearch.test.com' > data1/data1.ext
openssl x509 -req -in data1/data1.csr -CA root-ca.pem -CAkey root-ca.key -sha256 -out data1/data1.pem -days 800 -extfile data1/data1.ext
openssl genpkey -out dashboard/dashboard.key -algorithm RSA -pkeyopt rsa_keygen_bits:2048
openssl pkcs8 -inform PEM -outform PEM -in dashboard/dashboard.key -topk8 -nocrypt -v1 PBE-SHA1-3DES -out dashboard/dashboad_pkcs8.key
openssl req -new -key dashboard/dashboard_pkcs8.key -subj "/C=GB/L=London/O=test/CN=dashboard.opensearch.test.com" -out dashboard/dashboard.csr
echo 'subjectAltName=DNS:dashboard.opensearch.test.com' > dashboard/dashboard.ext
openssl x509 -req -in dashboard/dashboard.csr -CA root-ca.pem -CAkey root-ca.key -sha256 -out dashboard/dashboard.pem -days 800 -extfile dashboard/dashboard.ext
Configuration manager1:
cluster.name: opensearch-cluster
node.name: manager1.opensearch.test.com
network.host: 0.0.0.0
discovery.seed_hosts: [manager1.opensearch.test.com]
cluster.initial_cluster_manager_nodes: [manager1.opensearch.test.com]
node.roles: [cluster_manager]
plugins.security.disabled: false
plugins.security.allow_unsafe_democertificates: false
plugins.security.ssl.transport.pemcert_filepath: manager1.pem
plugins.security.ssl.transport.pemkey_filepath: manager1_pkcs8.key
plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.pem
plugins.security.ssl.transport.enforce_hostname_verification: false
plugins.security.ssl.http.enabled: true
plugins.security.ssl.http.pemcert_filepath: manager1.pem
plugins.security.ssl.http.pemkey_filepath: manager1_pkcs8.key
plugins.security.ssl.http.pemtrustedcas_filepath: root-ca.pem
plugins.security.allow_default_init_securityindex: true
plugins.security.authcz.admin_dn:
- 'CN=admin.test.com,O=test,L=London,C=GB'
plugins.security.nodes_dn:
- 'CN=manager1.opensearch.test.com,O=test,L=London,C=GB'
- 'CN=data1.opensearch.test.com,O=test,L=London,C=GB'
- 'CN=dashboard.opensearch.test.com,O=test,L=London,C=GB'
plugins.security.ssl.http.clientauth_mode: REQUIRE
plugins.security.audit.type: internal_opensearch
plugins.security.enable_snapshot_restore_privilege: true
plugins.security.check_snapshot_restore_write_privileges: true
plugins.security.restapi.roles_enabled: [all_access, security_rest_api_access]
plugins.security.system_indices.enabled: true
plugins.security.system_indices.indices: [.plugins-ml-agent, .plugins-ml-config, .plugins-ml-connector,
.plugins-ml-controller, .plugins-ml-model-group, .plugins-ml-model, .plugins-ml-task,
.plugins-ml-conversation-meta, .plugins-ml-conversation-interactions, .plugins-ml-memory-meta,
.plugins-ml-memory-message, .plugins-ml-stop-words, .opendistro-alerting-config,
.opendistro-alerting-alert*, .opendistro-anomaly-results*, .opendistro-anomaly-detector*,
.opendistro-anomaly-checkpoints, .opendistro-anomaly-detection-state, .opendistro-reports-*,
.opensearch-notifications-*, .opensearch-notebooks, .opensearch-observability, .ql-datasources,
.opendistro-asynchronous-search-response*, .replication-metadata-store, .opensearch-knn-models,
.geospatial-ip2geo-data*, .plugins-flow-framework-config, .plugins-flow-framework-templates,
.plugins-flow-framework-state]
Configuration dashboard:
opensearch.hosts: ["https://manager1.opensearch.test.com"]
opensearch.ssl.alwaysPresentCertificate: true
opensearch.ssl.verificationMode: full
opensearch.ssl.certificateAuthorities: [ "/usr/share/opensearch-dashboards/config/root-ca.pem" ]
opensearch.ssl.certificate: "/usr/share/opensearch-dashboards/config/dashboard.pem"
opensearch.ssl.key: "/usr/share/opensearch-dashboards/config/dashboard_pkcs8.key"
server.ssl.enabled: true
server.ssl.clientAuthentication: optional
server.ssl.certificateAuthorities: [ "/usr/share/opensearch-dashboards/config/root-ca.pem" ]
server.ssl.certificate: "/usr/share/opensearch-dashboards/config/dashboard.pem"
server.ssl.key: "/usr/share/opensearch-dashboards/config/dashboard_pkcs8.key"
opensearch.username: "kibanaserver"
opensearch.password: "some_passwd"
opensearch.requestHeadersWhitelist: [authorization, securitytenant]
opensearch_security.multitenancy.enabled: true
opensearch_security.multitenancy.tenants.preferred: [Private, Global]
opensearch_security.readonly_mode.roles: [kibana_read_only]
# Use this setting if you are running opensearch-dashboards without https
opensearch_security.cookie.secure: true
server.host: '0.0.0.0'
server.name: 'dashboard.opensearch.webscar.ru'
Configuration nginx of manager1:
upstream manager1 {
server 127.0.0.1:9200;
}
server {
listen 80;
server_name manager1.opensearch.test.com;
location / {
return 301 https://$host$request_uri;
}
}
server {
listen 443 ssl;
server_name manager1.opensearch.test.com;
ssl_certificate /opt/certs/manager1/fullchain_manager1.pem; // cat manager1.pem root-ca.pem > fullchain_manager1.pem
ssl_certificate_key /opt/certs/manager1/manager1.key;
access_log /var/log/nginx/manager1.opensearch.test_access.log;
error_log /var/log/nginx/manager1.opensearch.test_error.log;
location / {
proxy_pass https://manager1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}
Relevant Logs or Screenshots:
{"type":"log","@timestamp":"2024-04-23T11:08:21Z","tags":["error","opensearch","data"],"pid":1,"message":"[ConnectionError]: unable to get local issuer certificate"}