Versions (relevant - OpenSearch/Dashboard/Server OS/Browser):
Opensearch 2.20 and Opensearch Dashboards 2.18
Running in Openshift from official images
Describe the issue:
Hello! I want to deploy opensearch, all creds get from vault with webhook, I check, but I have error authentification failed for user? What do I wrong?
Configuration:
singleNode: {{ if gt (.Values.opensearch.replicas) 1 }}false{{else}}true{{end}}
replicas: {{ .Values.opensearch.replicas }}
image:
repository: “opensearchproject/opensearch”
tag: “2.14.0”
pullPolicy: “IfNotPresent”
ingress:
enabled: true
ingressClassName: {{ .Values.internalIngressClass }}
annotations:
cert-manager.io/cluster-issuer: letsencrypt-certificate
nginx.ingress.kubernetes.io/backend-protocol: “HTTPS”
nginx.ingress.kubernetes.io/proxy-body-size: “50m”
nginx.ingress.kubernetes.io/proxy-read-timeout: “30”
nginx.ingress.kubernetes.io/proxy-send-timeout: “30”
nginx.ingress.kubernetes.io/proxy-buffering: “off”
nginx.ingress.kubernetes.io/proxy-request-buffering: “off”
hosts:
- opensearch.{{ .Values.internalDomain }}
tls:
- secretName: opensearch-tls
hosts:
- opensearch.{{ .Values.internalDomain }}
opensearchJavaOpts: {{ .Values.opensearch | get “opensearchJavaOpts” “-Xmx512M -Xms512M” }}
updateStrategy: {{ .Values.opensearch | get “updateStrategy” “RollingUpdate”}}
extraEnvs:
- name: DISABLE_INSTALL_DEMO_CONFIG
value: “true”
config:
opensearch.yml: |
node.store.allow_mmap: false
cluster.name: {{ .Release.Name }}-cluster
network.host: 0.0.0.0
{{- if .Values.opensearch | get “s3” false }}
s3.client.default.region: {{ .Values.opensearch.s3.region }}
{{- end }}
plugins:
security:
nodes_dn:
- ‘CN=opensearchnode*’
ssl:
transport:
pemcert_filepath: node-tls/node1.pem
pemkey_filepath: node-tls/node1-key.pem
pemtrustedcas_filepath: node-tls/root-ca.pem
enforce_hostname_verification: false
http:
enabled: true
pemcert_filepath: node-tls/node1.pem
pemkey_filepath: node-tls/node1-key.pem
pemtrustedcas_filepath: node-tls/root-ca.pem
allow_default_init_securityindex: true
authcz:
admin_dn:
- ‘CN=admin,OU=devops,O=Arenadata,L=Moscow,C=RU’
audit.type: internal_opensearch
restapi:
roles_enabled: [“all_access”, “security_rest_api_access”]
system_indices:
enabled: true
indices:
[
“.opendistro-alerting-config”,
“.opendistro-alerting-alert*”,
“.opendistro-anomaly-results*”,
“.opendistro-anomaly-detector*”,
“.opendistro-anomaly-checkpoints”,
“.opendistro-anomaly-detection-state”,
“.opendistro-reports-",
".opendistro-notifications-”,
“.opendistro-notebooks”,
“.opendistro-asynchronous-search-response*”,
]
prometheus:
indices: false
terminationGracePeriod: {{ .Values.opensearch | get “terminationGracePeriod” 120 }}
persistence:
size: “8Gi”
storageClass: “csi-ceph-ssd-me1”
securityConfig:
actionGroupsSecret: {{ .Release.Name }}-config
configSecret: {{ .Release.Name }}-config
internalUsersSecret: {{ .Release.Name }}-config
rolesSecret: {{ .Release.Name }}-config
rolesMappingSecret: {{ .Release.Name }}-config
tenantsSecret: {{ .Release.Name }}-config
{{- with .Values.opensearch | get “resources” dict }}
resources: {{ . | toYaml | nindent 2}}
{{- end }}
{{- if not (.Values.opensearch | get “affinityDisabled” false) }}
antiAffinityTopologyKey: {{ .Values.topologyKeyZoneAffinity }}
antiAffinity: “hard”
{{- end }}
rbac:
create: true
serviceAccountName: vault-auth
automountServiceAccountToken: true
podAnnotations:
vault.security.banzaicloud.io/vault-role: “vault-secrets-webhook”
vault.security.banzaicloud.io/vault-skip-verify: “true”
vault.security.banzaicloud.io/vault-addr: “http://vault-server.vault:8200”
vault.security.banzaicloud.io/vault-agent: “false”
vault.security.banzaicloud.io/vault-path: “kubernetes”
prometheus.io/scrape-opensearch: “true”
extraObjects:
-
apiVersion: v1
kind: Secret
metadata:
name: {{ .Release.Name }}-config
namespace: {{ .Release.Namespace }}
annotations:
vault.security.banzaicloud.io/vault-role: “vault-secrets-webhook”
vault.security.banzaicloud.io/vault-skip-verify: “true”
vault.security.banzaicloud.io/vault-addr: “http://vault-server.vault:8200”
vault.security.banzaicloud.io/vault-agent: “false”
vault.security.banzaicloud.io/vault-path: “kubernetes”
type: Opaque
stringData:
config.yml: |
_meta:
type: “config”
config_version: 2
config:
dynamic:
http:
anonymous_auth_enabled: false
authc:
basic_internal_auth_domain:
http_enabled: true
transport_enabled: true
order: 0
http_authenticator:
type: basic
challenge: false
authentication_backend:
type: internalnodes_dn.yml: |-
_meta:
type: “nodesdn”
config_version: 2
whitelist.yml: |-
_meta:
type: “whitelist”
config_version: 2
tenants.yml: |-
_meta:
type: “tenants”
config_version: 2
admin_tenant:
reserved: false
description: “tenant for admin user”
action_groups.yml: |-
_meta:
type: “actiongroups”
config_version: 2
roles.yml: |-
_meta:
type: “roles”
config_version: 2
monitoring-role:
reserved: false
hidden: false
cluster_permissions:
- “cluster:monitor/prometheus/metrics”
- “cluster:monitor/health”
- “cluster:monitor/nodes/info”
- “cluster:monitor/nodes/stats”
- “cluster:monitor/state”
roles_mapping.yml: |-
_meta:
type: “rolesmapping”
config_version: 2
all_access:
reserved: false
backend_roles:
- “admin”
description: “Maps admin to all_access”
monitoring-role:
reserved: false
hidden: false
users:
- monitoring
internal_users.yml: |-
_meta:
type: “internalusers”
config_version: 2
admin:
hash: vault:dev/data/opensearch#hash
backend_roles:
- “admin”
description: “Admin user”
dashboarduser:
hash: vault:dev/data/opensearch#dashboardHash
description: “user for dashboard”
backend_roles:
- “admin”
monitoring:
hash: vault:dev/data/opensearch#monitoringHash
description: “user for monitoring” -
apiVersion: v1
kind: Secret
metadata:
name: admin-tls
namespace: {{ .Release.Namespace }}
annotations:
vault.security.banzaicloud.io/vault-role: “vault-secrets-webhook”
vault.security.banzaicloud.io/vault-skip-verify: “true”
vault.security.banzaicloud.io/vault-addr: “http://vault-server.vault:8200”
vault.security.banzaicloud.io/vault-agent: “false”
vault.security.banzaicloud.io/vault-path: “kubernetes”
type: Opaque
stringData:
admin-key.pem: vault:dev/data/opensearch#admin-key.pem
admin.pem: vault:dev/data/opensearch#admin.pem
root-ca-key.pem: vault:dev/data/opensearch#root-ca-key.pem
root-ca.pem: vault:dev/data/opensearch#root-ca.pem -
apiVersion: v1
kind: Secret
metadata:
name: node-tls
namespace: {{ .Release.Namespace }}
annotations:
vault.security.banzaicloud.io/vault-role: “vault-secrets-webhook”
vault.security.banzaicloud.io/vault-skip-verify: “true”
vault.security.banzaicloud.io/vault-addr: “http://vault-server.vault:8200”
vault.security.banzaicloud.io/vault-agent: “false”
vault.security.banzaicloud.io/vault-path: “kubernetes”
type: Opaque
stringData:
node1-key.pem: vault:dev/data/opensearch#node1-key.pem
node1.pem: vault:dev/data/opensearch#node1.pem
root-ca-key.pem: vault:dev/data/opensearch#root-ca-key.pem
root-ca.pem: vault:dev/data/opensearch#root-ca.pem
— opensearch-dashboard
image:
repository: “opensearchproject/opensearch-dashboards”
tag: “2.14.0”
ingress:
enabled: true
ingressClassName: {{ .Values.internalIngressClass }}
annotations:
nginx.ingress.kubernetes.io/force-ssl-redirect: “true”
hosts:
- host: logs.{{ .Values.internalDomain }}
paths:
- path: /
backend:
serviceName: “”
servicePort: “”
tls:
- secretName: logs-tls
hosts:
- logs.{{ .Values.internalDomain }}
serviceAccount:
create: false
name: vault-auth
podAnnotations:
vault.security.banzaicloud.io/vault-role: “vault-secrets-webhook”
vault.security.banzaicloud.io/vault-skip-verify: “true”
vault.security.banzaicloud.io/vault-addr: “http://vault-server.vault:8200”
vault.security.banzaicloud.io/vault-agent: “false”
vault.security.banzaicloud.io/vault-path: “kubernetes”
extraEnvs:
- name: OPENSEARCH_USERNAME
value: “dashboarduser” - name: OPENSEARCH_PASSWORD
value: {{ .Values.opensearch.dashboardPassword | quote }}
resources:
limits:
cpu: “1000m”
memory: “1G”
extraVolumeMounts:
- name: config
mountPath: /usr/share/opensearch-dashboards/config/opensearch_dashboards.yml
subPath: opensearch_dashboards.yml
extraVolumes:
- name: config
secret:
secretName: config
extraObjects:
- apiVersion: v1
kind: Secret
metadata:
name: config
namespace: {{ .Release.Namespace }}
annotations:
vault.security.banzaicloud.io/vault-role: “vault-secrets-webhook”
vault.security.banzaicloud.io/vault-skip-verify: “true”
vault.security.banzaicloud.io/vault-addr: “http://vault-server.vault:8200”
vault.security.banzaicloud.io/vault-agent: “false”
vault.security.banzaicloud.io/vault-path: “kubernetes”
type: Opaque
stringData:
opensearch_dashboards.yml: |
opensearch_security.auth.type: [“basicauth”]
opensearch.ssl.verificationMode: none
opensearch.requestHeadersAllowlist: [“Authorization”, “securitytenant”]
opensearch_security.cookie.ttl: 46400000
opensearch_security.session.ttl: 46400000
opensearch_security.session.keepalive: true
opensearch.username: “dashboarduser”
opensearch.password: vault:dev/data/opensearch#dashboardPassword
sysctlVmMaxMapCount: 262144
Relevant Logs or Screenshots:
[WARN ][o.o.s.a.BackendRegistry ] [opensearch-cluster-master-0] Authentication finally failed for dashboarduser from 10.100.83.226:36168 -----log from opensearch-cluster
{“type”:“log”,“@timestamp”:“2024-09-23T08:26:47Z”,“tags”:[“error”,“opensearch”,“data”],“pid”:1,“message”:“[ResponseError]: Response Error”} ----log from opensearch-dashboard