Transport Client Authentication Error

Issue

## Versions (relevant - OpenSearch/Dashboard/Server OS/Browser):
- OpenSearch Version: 2.15.0

## Describe the issue:

We are experiencing an issue with the OpenSearch cluster where the transport client authentication is no longer supported, leading to repeated `OpenSearchException` errors in the log files.

## Configuration:

Here is the relevant portion of our configuration file:
---
clusterName: "opensearch-cluster"
nodeGroup: "master"
discovery.seed_hosts: ["opensearch-cluster-master-0", "opensearch-cluster-master-1"]
singleNode: false

masterService: "opensearch-cluster-master"

roles:
  - cluster_manager
  - ingest
  - data
  - remote_cluster_client

replicas: 2

global:
  dockerRegistry: ""

opensearchHome: /usr/share/opensearch

config:
  opensearch.yml: |
    cluster.name: opensearch-cluster
    http.host: "0.0.0.0"
    # Configurazione SSL/TLS per il trasporto
    plugins.security.ssl.transport.enabled: true
    plugins.security.ssl.transport.pemcert_filepath: /usr/share/opensearch/config/certs/node1.pem
    plugins.security.ssl.transport.pemkey_filepath: /usr/share/opensearch/config/certs/node1-key.pem
    plugins.security.ssl.transport.pemtrustedcas_filepath: /usr/share/opensearch/config/certs/root-ca-node1.pem

    # Configurazione SSL/TLS per HTTP
    plugins.security.ssl.http.enabled: true
    plugins.security.ssl.http.pemcert_filepath: /usr/share/opensearch/config/certs/node1.pem
    plugins.security.ssl.http.pemkey_filepath: /usr/share/opensearch/config/certs/node1-key.pem
    plugins.security.ssl.http.pemtrustedcas_filepath: /usr/share/opensearch/config/certs/root-ca-node1.pem

    # # Configurazione dei DN autorizzati
    plugins.security.nodes_dn:
      - "CN=opensearch-cluster-master:9300,OU=..., O=..., L=..., ST=..., C=..."

    # Altre impostazioni di sicurezza
    plugins.security.allow_unsafe_democertificates: true
    plugins.security.allow_default_init_securityindex: true
    plugins.security.authcz.admin_dn:
      - "CN=ADMIN, OU=..., O=..., L=..., ST=..., C=..."
    plugins.security.audit.type: internal_opensearch
    plugins.security.enable_snapshot_restore_privilege: true
    plugins.security.check_snapshot_restore_write_privileges: true
    plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"]
    plugins.security.system_indices.enabled: true
    plugins.security.system_indices.indices: [ ".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opendistro-notifications-*", ".opendistro-notebooks", ".opendistro-asynchronous-search-response*"]
    logger.org.opensearch.transport: DEBUG
    logger.org.opensearch.security: DEBUG

extraEnvs: 
  - name: "OPENSEARCH_INITIAL_ADMIN_PASSWORD"
    value: "<REDACTED>"
  - name: DISABLE_INSTALL_DEMO_CONFIG
    value: "true"

envFrom: []

secretMounts: 
  - name: open-certificates
    secretName: open-certificates
    path: /usr/share/opensearch/config/certs/

hostAliases: []

image:
  repository: "opensearchproject/opensearch"
  tag: "2.15.0"
  pullPolicy: "IfNotPresent"

podAnnotations: {}

openSearchAnnotations: {}

labels: {}

opensearchJavaOpts: "-Xmx512M -Xms512M"

resources:
  requests:
    cpu: "1000m"
    memory: "100Mi"

initResources: {}

sidecarResources: {}

networkHost: "0.0.0.0"

rbac:
  create: false
  serviceAccountAnnotations: {}
  serviceAccountName: ""
  automountServiceAccountToken: false

podSecurityPolicy:
  create: false
  name: ""
  spec:
    privileged: true
    fsGroup:
      rule: RunAsAny
    runAsUser:
      rule: RunAsAny
    seLinux:
      rule: RunAsAny
    supplementalGroups:
      rule: RunAsAny
    volumes:
      - secret
      - configMap
      - persistentVolumeClaim
      - emptyDir

persistence:
  enabled: true
  enableInitChown: true
  labels:
    enabled: false
  accessModes:
    - ReadWriteOnce
  size: 8Gi
  annotations: {}

extraVolumes: []

extraVolumeMounts: []

extraContainers: []

extraInitContainers: []

priorityClassName: ""

antiAffinityTopologyKey: "kubernetes.io/hostname"
antiAffinity: "soft"
customAntiAffinity: {}

nodeAffinity: {}

podAffinity: {}

topologySpreadConstraints: []

podManagementPolicy: "Parallel"
enableServiceLinks: true

protocol: https
httpPort: 9200
transportPort: 9300
metricsPort: 9600
httpHostPort: ""
transportHostPort: ""

service:
  labels: {}
  labelsHeadless: {}
  headless:
    annotations: {}
  type: ClusterIP
  nodePort: ""
  annotations: {}
  httpPortName: http
  transportPortName: transport
  metricsPortName: metrics
  loadBalancerIP: ""
  loadBalancerSourceRanges: []
  externalTrafficPolicy: ""

updateStrategy: RollingUpdate
maxUnavailable: 1

podSecurityContext:
  fsGroup: 1000
  runAsUser: 1000

securityContext:
  capabilities:
    drop:
      - ALL
  runAsNonRoot: true
  runAsUser: 1000

securityConfig:
  enabled: true
  path: "/usr/share/opensearch/config/opensearch-security"
  config:
    data: 
      config.yml: |
        _meta:
          type: "config"
          config_version: 2
        config:
          dynamic:
            authc:
              basic_internal_auth_domain:
                http_enabled: true
                transport_enabled: true
                order: 0
                http_authenticator:
                  type: basic
                  challenge: false
                authentication_backend:
                  type: internal
              openid_auth_domain:
                http_enabled: true
                transport_enabled: true
                order: 1
                http_authenticator:
                  type: openid
                  challenge: false
                  config:
                    subject_key: admin
                    openid_connect_url: https://keycloak.example.com/realms/master/.well-known/openid-configuration
                authentication_backend:
                  type: noop

terminationGracePeriod: 120

sysctlVmMaxMapCount: 262144

startupProbe:
  tcpSocket:
    port: 9200
  initialDelaySeconds: 5
  periodSeconds: 10
  timeoutSeconds: 3
  failureThreshold: 30

livenessProbe: {}

readinessProbe:
  tcpSocket:
    port: 9200
  periodSeconds: 5
  timeoutSeconds: 3
  failureThreshold: 3

schedulerName: ""

imagePullSecrets: []
nodeSelector: {}
tolerations: []

ingress:
  enabled: false
  ingressClassName: nginx
  path: /
  hosts:
    - opensearch.example.local
  tls:
    - hosts:
      - opensearch.example.local
  annotations: 
    nginx.ingress.kubernetes.io/proxy-connect-timeout: "60s"
    nginx.ingress.kubernetes.io/proxy-read-timeout: "600s"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "600s"
    nginx.ingress.kubernetes.io/proxy-buffering: "on"
    nginx.ingress.kubernetes.io/proxy-buffer-size: "64k"

nameOverride: ""
fullnameOverride: ""

masterTerminationFix: false

opensearchLifecycle: {}

lifecycle: {}

keystore: []

networkPolicy:
  create: false
  http:
    enabled: false

sysctl:
  enabled: false

sysctlInit:
  enabled: false

plugins:
  enabled: true
  installList: []

extraObjects: []

Relevant Logs


## Relevant Logs or Screenshots:

Below are the error logs indicating the issue:

```plaintext
opensearch [2024-07-22T12:07:33,202][ERROR][o.o.s.t.SecurityRequestHandler] [opensearch-cluster-master-1] OpenSearchException[Transport client authentication no longer supported.]
opensearch [2024-07-22T12:07:34,202][ERROR][o.o.s.t.SecurityRequestHandler] [opensearch-cluster-master-1] OpenSearchException[Transport client authentication no longer supported.]
opensearch [2024-07-22T12:07:35,205][ERROR][o.o.s.t.SecurityRequestHandler] [opensearch-cluster-master-1] OpenSearchException[Transport client authentication no longer supported.]
opensearch [2024-07-22T12:07:36,204][ERROR][o.o.s.t.SecurityRequestHandler] [opensearch-cluster-master-1] OpenSearchException[Transport client authentication no longer supported.]
opensearch [2024-07-22T12:07:37,203][ERROR][o.o.s.t.SecurityRequestHandler] [opensearch-cluster-master-1] OpenSearchException[Transport client authentication no longer supported.]
opensearch [2024-07-22T12:07:38,205][ERROR][o.o.s.t.SecurityRequestHandler] [opensearch-cluster-master-1] OpenSearchException[Transport client authentication no longer supported.]
opensearch [2024-07-22T12:07:39,204][ERROR][o.o.s.t.SecurityRequestHandler] [opensearch-cluster-master-1] OpenSearchException[Transport client authentication no longer supported.]
opensearch [2024-07-22T12:07:40,203][ERROR][o.o.s.t.SecurityRequestHandler] [opensearch-cluster-master-1] OpenSearchException[Transport client authentication no longer supported.]
opensearch [2024-07-22T12:07:41,110][DEBUG][o.o.s.c.ConfigurationRepository] [opensearch-cluster-master-1] Unable to load configuration due to java.util.concurrent.TimeoutException: Timeout after 10SECONDS while retrieving configuration for [ACTIONGROUPS, ALLOWLIST, AUDIT, CONFIG, INTERNALUSERS, NODESDN, ROLES, ROLESMAPPING, TENANTS, WHITELIST](index=.opendistro_security)

I tried everything but i can’t fix this error. Can someone help me please?

1 Like

Generally, the error log you’ve shown happens when connecting each node through Transport Layer fails.

  1. How did you issue the certificates (.pem)?
  2. Does your container allow you to access in and curl -XGET https://localhost:9200 ?

Hi, thank you for your response.

I followed the official documentation for creating the certificates: OpenSearch Certificate Generation Guide.

Here are the commands I used:

# Root CA
openssl genrsa -out root-ca-key.pem 2048
openssl req -new -x509 -sha256 -key root-ca-key.pem -subj "/C=../ST=../L=../O=../OU=../CN=opensearch-cluster-master-headless" -out root-ca.pem -days 2048

# Node cert 1
openssl genrsa -out node1-key-temp.pem 2048
openssl pkcs8 -inform PEM -outform PEM -in node1-key-temp.pem -topk8 -nocrypt -v1 PBE-SHA1-3DES -out node1-key.pem
openssl req -new -key node1-key.pem -subj "/C=../ST=../L=../O=../OU=../CN=opensearch-cluster-master-headless" -out node1.csr
echo 'subjectAltName=DNS:opensearch-cluster-master-headless' > node1.ext
openssl x509 -req -in node1.csr -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial -sha256 -out node1.pem -days 2048 -extfile node1.ext

When I run the curl command from within the container, I get the following result:

[opensearch@opensearch-cluster-master-0 ~]$ curl -XGET https://localhost:9200 -k
OpenSearch Security not initialized.

It seems like the security module in OpenSearch isn’t initialized. Could you provide guidance on what might be going wrong or how to properly initialize the security settings?

Thank you!

1 Like

I also tried using the following command:


[opensearch@opensearch-cluster-master-0 tools]$ ./securityadmin.sh -cd /usr/share/opensearch/config/certs/ -icl -nhnv -cacert /usr/share/opensearch/config/certs/root-ca-node1.pem -cert /usr/share/opensearch/config/certs/node1.pem -key /usr/share/opensearch/config/certs/node1-key.pem
**************************************************************************
** This tool will be deprecated in the next major release of OpenSearch **
** https://github.com/opensearch-project/security/issues/1755           **
**************************************************************************
Security Admin v7
Will connect to localhost:9200 ... done
Connected as "CN=opensearch-cluster-master-headless,OU=...,O=...,L=...,ST=...,C=..."
ERR: "CN=opensearch-cluster-master-headless,OU=...,O=...,L=...,ST=...,C=..." is not an admin user
Seems you use a client certificate but this one is not registered as admin_dn
Make sure opensearch.yml on all nodes contains:
plugins.security.authcz.admin_dn:
  - ""CN=opensearch-cluster-master-headless,OU=...,O=...,L=...,ST=...,C=...""

I understand that I should have certificates for the admin to use in authcz.admin_dn and separate certificates for the nodes to use in nodes_dn. The problem arises when I configure the fields for plugins.security.ssl.transport and plugins.security.ssl.http, as I can only pass one of the two types of certificates to them. If I understand correctly, I need a solution for this issue.

1 Like

Generally, OpenSearch Cluster starts after .opendistro_security index created. It can be easily created by securityadmin.sh located in /usr/share/opensearch/plugins/opensearch-security/tools.

In your manifest file, there isn’t any information about security except config.yml.

I currently created OpenSearch Cluster using the operator and this way is not much different with Helm deployment.
By mounting two Secrets (securityconfig-secret and admin-credentials-secret) as volume,

  volumeMounts:
    - mountPath: >-
        /usr/share/opensearch/config/opensearch-security/action_groups.yml
      name: securityconfig
      readOnly: true
      subPath: action_groups.yml
    - mountPath: /usr/share/opensearch/config/opensearch-security/config.yml
      name: securityconfig
      readOnly: true
      subPath: config.yml
    ...
    - mountPath: /usr/share/opensearch/data
      name: data
    - mountPath: /mnt/admin-credentials
      name: admin-credentials

Job(Kubernetes api resource) named securityconfig-update running as a Container.


The OpenSearch Operator recommends(example) me to include action_groups.yml, internal_users.yml, nodes_dn.yml, whitelist.yml, tenants.yml, roles_mapping.yml, roles.yml, and config.yml (same as yours).

apiVersion: v1
kind: Secret
metadata:
  name: securityconfig-secret
  namespace: test-opensearch-cluster
type: Opaque
## admin opassword hash for test "$2y$12$B6GMBQIwOUEV2qtBQrpJL.37MUMp1XkLxCyWzeTH5Q94QxNjw8ng6"
stringData:
      action_groups.yml: |-
         _meta:
           type: "actiongroups"
           config_version: 2
      internal_users.yml: |-
        _meta:
          type: "internalusers"
          config_version: 2
        admin:
          hash: "$2y$12$tS0wrbNssQpVjOXDPrzqdO5phJC/Fmb9fNKSdJ9P2voGK.LNIqLxG"
          reserved: true
          backend_roles:
          - "admin"
          description: "Demo admin user"
        dashboarduser:
          hash: "$2a$12$4AcgAt3xwOWadA5s5blL6ev39OXDNhmOesEoo33eZtrq2N0YrU3H."
          reserved: true
          description: "Demo OpenSearch Dashboards user"
      nodes_dn.yml: |-
        _meta:
          type: "nodesdn"
          config_version: 2
      whitelist.yml: |-
        _meta:
          type: "whitelist"
          config_version: 2
      tenants.yml: |-
        _meta:
          type: "tenants"
          config_version: 2
      roles_mapping.yml: |-
        _meta:
          type: "rolesmapping"
          config_version: 2
        all_access:
          reserved: false
          backend_roles:
          - "admin"
          description: "Maps admin to all_access"
        own_index:
          reserved: false
          users:
          - "*"
          description: "Allow full access to an index named like the username"
        readall:
          reserved: false
          backend_roles:
          - "readall"
        manage_snapshots:
          reserved: false
          backend_roles:
          - "snapshotrestore"
        dashboard_server:
          reserved: true
          users:
          - "dashboarduser"
      roles.yml: |-
        _meta:
          type: "roles"
          config_version: 2
        dashboard_read_only:
          reserved: true
        security_rest_api_access:
          reserved: true
        # Allows users to view monitors, destinations and alerts
        alerting_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/alerting/alerts/get'
            - 'cluster:admin/opendistro/alerting/destination/get'
            - 'cluster:admin/opendistro/alerting/monitor/get'
            - 'cluster:admin/opendistro/alerting/monitor/search'
        # Allows users to view and acknowledge alerts
        alerting_ack_alerts:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/alerting/alerts/*'
        # Allows users to use all alerting functionality
        alerting_full_access:
          reserved: true
          cluster_permissions:
            - 'cluster_monitor'
            - 'cluster:admin/opendistro/alerting/*'
          index_permissions:
            - index_patterns:
                - '*'
              allowed_actions:
                - 'indices_monitor'
                - 'indices:admin/aliases/get'
                - 'indices:admin/mappings/get'
        # Allow users to read Anomaly Detection detectors and results
        anomaly_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/ad/detector/info'
            - 'cluster:admin/opendistro/ad/detector/search'
            - 'cluster:admin/opendistro/ad/detectors/get'
            - 'cluster:admin/opendistro/ad/result/search'
            - 'cluster:admin/opendistro/ad/tasks/search'
            - 'cluster:admin/opendistro/ad/detector/validate'
            - 'cluster:admin/opendistro/ad/result/topAnomalies'
        # Allows users to use all Anomaly Detection functionality
        anomaly_full_access:
          reserved: true
          cluster_permissions:
            - 'cluster_monitor'
            - 'cluster:admin/opendistro/ad/*'
          index_permissions:
            - index_patterns:
                - '*'
              allowed_actions:
                - 'indices_monitor'
                - 'indices:admin/aliases/get'
                - 'indices:admin/mappings/get'
        # Allows users to read Notebooks
        notebooks_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/notebooks/list'
            - 'cluster:admin/opendistro/notebooks/get'
        # Allows users to all Notebooks functionality
        notebooks_full_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/notebooks/create'
            - 'cluster:admin/opendistro/notebooks/update'
            - 'cluster:admin/opendistro/notebooks/delete'
            - 'cluster:admin/opendistro/notebooks/get'
            - 'cluster:admin/opendistro/notebooks/list'
        # Allows users to read observability objects
        observability_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opensearch/observability/get'
        # Allows users to all Observability functionality
        observability_full_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opensearch/observability/create'
            - 'cluster:admin/opensearch/observability/update'
            - 'cluster:admin/opensearch/observability/delete'
            - 'cluster:admin/opensearch/observability/get'
        # Allows users to read and download Reports
        reports_instances_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/reports/instance/list'
            - 'cluster:admin/opendistro/reports/instance/get'
            - 'cluster:admin/opendistro/reports/menu/download'
        # Allows users to read and download Reports and Report-definitions
        reports_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/reports/definition/get'
            - 'cluster:admin/opendistro/reports/definition/list'
            - 'cluster:admin/opendistro/reports/instance/list'
            - 'cluster:admin/opendistro/reports/instance/get'
            - 'cluster:admin/opendistro/reports/menu/download'
        # Allows users to all Reports functionality
        reports_full_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/reports/definition/create'
            - 'cluster:admin/opendistro/reports/definition/update'
            - 'cluster:admin/opendistro/reports/definition/on_demand'
            - 'cluster:admin/opendistro/reports/definition/delete'
            - 'cluster:admin/opendistro/reports/definition/get'
            - 'cluster:admin/opendistro/reports/definition/list'
            - 'cluster:admin/opendistro/reports/instance/list'
            - 'cluster:admin/opendistro/reports/instance/get'
            - 'cluster:admin/opendistro/reports/menu/download'
        # Allows users to use all asynchronous-search functionality
        asynchronous_search_full_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/asynchronous_search/*'
          index_permissions:
            - index_patterns:
                - '*'
              allowed_actions:
                - 'indices:data/read/search*'
        # Allows users to read stored asynchronous-search results
        asynchronous_search_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/asynchronous_search/get'
        # Allows user to use all index_management actions - ism policies, rollups, transforms
        index_management_full_access:
          reserved: true
          cluster_permissions:
            - "cluster:admin/opendistro/ism/*"
            - "cluster:admin/opendistro/rollup/*"
            - "cluster:admin/opendistro/transform/*"
          index_permissions:
            - index_patterns:
                - '*'
              allowed_actions:
                - 'indices:admin/opensearch/ism/*'
        # Allows users to use all cross cluster replication functionality at leader cluster
        cross_cluster_replication_leader_full_access:
          reserved: true
          index_permissions:
            - index_patterns:
                - '*'
              allowed_actions:
                - "indices:admin/plugins/replication/index/setup/validate"
                - "indices:data/read/plugins/replication/changes"
                - "indices:data/read/plugins/replication/file_chunk"
        # Allows users to use all cross cluster replication functionality at follower cluster
        cross_cluster_replication_follower_full_access:
          reserved: true
          cluster_permissions:
            - "cluster:admin/plugins/replication/autofollow/update"
          index_permissions:
            - index_patterns:
                - '*'
              allowed_actions:
                - "indices:admin/plugins/replication/index/setup/validate"
                - "indices:data/write/plugins/replication/changes"
                - "indices:admin/plugins/replication/index/start"
                - "indices:admin/plugins/replication/index/pause"
                - "indices:admin/plugins/replication/index/resume"
                - "indices:admin/plugins/replication/index/stop"
                - "indices:admin/plugins/replication/index/update"
                - "indices:admin/plugins/replication/index/status_check"
      config.yml: |-
        _meta:
          type: "config"
          config_version: "2"
        config:
          dynamic:
            http:
              anonymous_auth_enabled: false
            authc:
              basic_internal_auth_domain:
                http_enabled: true
                transport_enabled: true
                order: "4"
                http_authenticator:
                  type: basic
                  challenge: true
                authentication_backend:
                  type: intern

In addition, I made a Secret(admin-credentials-secret) kind of Kubernetes API resource.

apiVersion: v1
kind: Secret
metadata:  
  name: admin-credentials-secret
  namespace: test-opensearch-cluster
type: Opaque
data:
  # admin
  username: YWRtaW4=
  # test
  password: dGVzdA==

As you suggested, I have added the security configuration in my Helm values:

securityConfig:
  enabled: true
  path: "/usr/share/opensearch/config/opensearch-security"
  config:
    data: 
      action_groups.yml: |-
         _meta:
           type: "actiongroups"
           config_version: 2
      internal_users.yml: |-
        _meta:
          type: "internalusers"
          config_version: 2
        admin:
          hash: "$2y$12$tS0wrbNssQpVjOXDPrzqdO5phJC/Fmb9fNKSdJ9P2voGK.LNIqLxG"
          reserved: true
          backend_roles:
          - "admin"
          description: "Demo admin user"
        dashboarduser:
          hash: "$2a$12$4AcgAt3xwOWadA5s5blL6ev39OXDNhmOesEoo33eZtrq2N0YrU3H."
          reserved: true
          description: "Demo OpenSearch Dashboards user"
      nodes_dn.yml: |-
        _meta:
          type: "nodesdn"
          config_version: 2
      whitelist.yml: |-
        _meta:
          type: "whitelist"
          config_version: 2
      tenants.yml: |-
        _meta:
          type: "tenants"
          config_version: 2
      roles_mapping.yml: |-
        _meta:
          type: "rolesmapping"
          config_version: 2
        all_access:
          reserved: false
          backend_roles:
          - "admin"
          description: "Maps admin to all_access"
        own_index:
          reserved: false
          users:
          - "*"
          description: "Allow full access to an index named like the username"
        readall:
          reserved: false
          backend_roles:
          - "readall"
        manage_snapshots:
          reserved: false
          backend_roles:
          - "snapshotrestore"
        dashboard_server:
          reserved: true
          users:
          - "dashboarduser"
      roles.yml: |-
        _meta:
          type: "roles"
          config_version: 2
        dashboard_read_only:
          reserved: true
        security_rest_api_access:
          reserved: true
        alerting_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/alerting/alerts/get'
            - 'cluster:admin/opendistro/alerting/destination/get'
            - 'cluster:admin/opendistro/alerting/monitor/get'
            - 'cluster:admin/opendistro/alerting/monitor/search'
        alerting_ack_alerts:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/alerting/alerts/*'
        alerting_full_access:
          reserved: true
          cluster_permissions:
            - 'cluster_monitor'
            - 'cluster:admin/opendistro/alerting/*'
          index_permissions:
            - index_patterns:
                - '*'
              allowed_actions:
                - 'indices_monitor'
                - 'indices:admin/aliases/get'
                - 'indices:admin/mappings/get'
        anomaly_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/ad/detector/info'
            - 'cluster:admin/opendistro/ad/detector/search'
            - 'cluster:admin/opendistro/ad/detectors/get'
            - 'cluster:admin/opendistro/ad/result/search'
            - 'cluster:admin/opendistro/ad/tasks/search'
            - 'cluster:admin/opendistro/ad/detector/validate'
            - 'cluster:admin/opendistro/ad/result/topAnomalies'
        anomaly_full_access:
          reserved: true
          cluster_permissions:
            - 'cluster_monitor'
            - 'cluster:admin/opendistro/ad/*'
          index_permissions:
            - index_patterns:
                - '*'
              allowed_actions:
                - 'indices_monitor'
                - 'indices:admin/aliases/get'
                - 'indices:admin/mappings/get'
        notebooks_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/notebooks/list'
            - 'cluster:admin/opendistro/notebooks/get'
        notebooks_full_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/notebooks/create'
            - 'cluster:admin/opendistro/notebooks/update'
            - 'cluster:admin/opendistro/notebooks/delete'
            - 'cluster:admin/opendistro/notebooks/get'
            - 'cluster:admin/opendistro/notebooks/list'
        observability_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opensearch/observability/get'
        observability_full_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opensearch/observability/create'
            - 'cluster:admin/opensearch/observability/update'
            - 'cluster:admin/opensearch/observability/delete'
            - 'cluster:admin/opensearch/observability/get'
        reports_instances_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/reports/instance/list'
            - 'cluster:admin/opendistro/reports/instance/get'
            - 'cluster:admin/opendistro/reports/menu/download'
        reports_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/reports/definition/get'
            - 'cluster:admin/opendistro/reports/definition/list'
            - 'cluster:admin/opendistro/reports/instance/list'
            - 'cluster:admin/opendistro/reports/instance/get'
            - 'cluster:admin/opendistro/reports/menu/download'
        reports_full_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/reports/definition/create'
            - 'cluster:admin/opendistro/reports/definition/update'
            - 'cluster:admin/opendistro/reports/definition/on_demand'
            - 'cluster:admin/opendistro/reports/definition/delete'
            - 'cluster:admin/opendistro/reports/definition/get'
            - 'cluster:admin/opendistro/reports/definition/list'
            - 'cluster:admin/opendistro/reports/instance/list'
            - 'cluster:admin/opendistro/reports/instance/get'
            - 'cluster:admin/opendistro/reports/menu/download'
        asynchronous_search_full_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/asynchronous_search/*'
          index_permissions:
            - index_patterns:
                - '*'
              allowed_actions:
                - 'indices:data/read/search*'
        asynchronous_search_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/asynchronous_search/get'
        index_management_full_access:
          reserved: true
          cluster_permissions:
            - "cluster:admin/opendistro/ism/*"
            - "cluster:admin/opendistro/rollup/*"
            - "cluster:admin/opendistro/transform/*"
          index_permissions:
            - index_patterns:
                - '*'
              allowed_actions:
                - 'indices:admin/opensearch/ism/*'
        cross_cluster_replication_leader_full_access:
          reserved: true
          index_permissions:
            - index_patterns:
                - '*'
              allowed_actions:
                - "indices:admin/plugins/replication/index/setup/validate"
                - "indices:data/read/plugins/replication/changes"
                - "indices:data/read/plugins/replication/file_chunk"
        cross_cluster_replication_follower_full_access:
          reserved: true
          cluster_permissions:
            - "cluster:admin/plugins/replication/autofollow/update"
          index_permissions:
            - index_patterns:
                - '*'
              allowed_actions:
                - "indices:admin/plugins/replication/index/setup/validate"
                - "indices:data/write/plugins/replication/changes"
                - "indices:admin/plugins/replication/index/start"
                - "indices:admin/plugins/replication/index/pause"
                - "indices:admin/plugins/replication/index/resume"
                - "indices:admin/plugins/replication/index/stop"
                - "indices:admin/plugins/replication/index/update"
                - "indices:admin/plugins/replication/index/status_check"
      config.yml: |-
        _meta:
          type: "config"
          config_version: 2
        config:
          dynamic:
            authc:
              basic_internal_auth_domain:
                http_enabled: true
                transport_enabled: true
                order: 0
                http_authenticator:
                  type: basic
                  challenge: false
                authentication_backend:
                  type: internal
              openid_auth_domain:
                http_enabled: true
                transport_enabled: true
                order: 1
                http_authenticator:
                  type: openid
                  challenge: false
                  config:
                    subject_key: admin
                    openid_connect_url: https://keycloak.example.com/realms/master/.well-known/openid-configuration
                authentication_backend:
                  type: noop

I also created a Job to execute these two commands:

  • curl -u admin:test -k --silent "https://opensearch-cluster-master.os-stack.svc.cluster.local:9200"
  • $ADMIN -cacert /usr/share/opensearch/config/certs/ca.crt -cert /usr/share/opensearch/config/certs/certs/tls.crt -key /usr/share/opensearch/config/certs/certs/tls.key -cd /usr/share/opensearch/config/opensearch-security -icl -nhnv -h https://opensearch-cluster-master.os-stack.svc.cluster.local -p 9200

Here is the manifest of the Job:

apiVersion: batch/v1
kind: Job
metadata:
  name: securityconfig-update
  namespace: os-stack
spec:
  template:
    spec:
      containers:
        - name: securityconfig-update
          image: opensearchproject/opensearch:2.15.0
          command: ["/bin/bash", "-c"]
          args: 
            - |
              ADMIN=/usr/share/opensearch/plugins/opensearch-security/tools/securityadmin.sh;
              chmod +x $ADMIN;
              until curl -u admin:test -k --silent "https://opensearch-cluster-master.os-stack.svc.cluster.local:9200" ; do
                echo 'Waiting to connect to the cluster'; sleep 120;
              done;
              count=0;
              until $ADMIN -cacert /usr/share/opensearch/config/certs/ca.crt -cert /usr/share/opensearch/config/certs/certs/tls.crt -key /usr/share/opensearch/config/certs/certs/tls.key -cd /usr/share/opensearch/config/opensearch-security -icl -nhnv -h https://opensearch-cluster-master.os-stack.svc.cluster.local -p 9200 || (( count++ >= 20 )); do

                sleep 20;
              done;
          volumeMounts:
            - name: securityconfig
              mountPath: /usr/share/opensearch/config/opensearch-security
              readOnly: true
            - name: admin-credentials
              mountPath: /certs
              readOnly: true
      volumes:
        - name: securityconfig
          secret:
            secretName: securityconfig-secret
        - name: admin-credentials
          secret:
            secretName: admin-credentials-secret
      restartPolicy: OnFailure

However, the job fails with the following connection error:

**************************************************************************
** This tool will be deprecated in the next major release of OpenSearch **
** https://github.com/opensearch-project/security/issues/1755           **
**************************************************************************
Security Admin v7
Will connect to https://opensearch-cluster-master.os-stack.svc.cluster.local:9200ERR: An unexpected UnknownHostException occured: https://opensearch-clu
Trace:
java.net.UnknownHostException: https://opensearch-cluster-master.os-stack.svc.cluster.local
    at java.base/sun.nio.ch.NioSocketImpl.connect(NioSocketImpl.java:567)
    at java.base/java.net.SocksSocketImpl.connect(SocksSocketImpl.java:327)
    at java.base/java.net.Socket.connect(Socket.java:751)
    at java.base/java.net.Socket.connect(Socket.java:686)
    at org.opensearch.security.tools.SecurityAdmin.execute(SecurityAdmin.java:521)
    at org.opensearch.security.tools.SecurityAdmin.main(SecurityAdmin.java:162)

When I run the same first command from the container shell, it succeeds:

[opensearch@securityconfig-update-c5z8v ~]$ curl -u admin:test -k --silent "https://opensearch-cluster-master.os-stack.svc.cluster.local:9200"
{
  "name" : "opensearch-cluster-master-1",
  "cluster_name" : "opensearch-cluster",
  "cluster_uuid" : "mD5n-tasQiiwxgMk4-Xywg",
  "version" : {
    "distribution" : "opensearch",
    "number" : "2.15.0",
    "build_type" : "tar",
    "build_hash" : "61dbcd0795c9bfe9b81e5762175414bc38bbcadf",
    "build_date" : "2024-06-20T03:26:49.193630411Z",
    "build_snapshot" : false,
    "lucene_version" : "9.10.0",
    "minimum_wire_compatibility_version" : "7.10.0",
    "minimum_index_compatibility_version" : "7.0.0"
  },
  "tagline" : "The OpenSearch Project: https://opensearch.org/"
}

However, the second command fails with the same error from the shell:

[opensearch@securityconfig-update-c5z8v ~]$ $ADMIN -cacert /certs/ca.crt -cert /certs/tls.crt -key /certs/tls.key -cd /usr/share/opensearch/config/opensearch-security -icl -nhnv -h https://opensearch-cluster-master.os-stack.svc.cluster.local -p 9200
**************************************************************************
** This tool will be deprecated in the next major release of OpenSearch **
** https://github.com/opensearch-project/security/issues/1755           **
**************************************************************************
Security Admin v7
Will connect to https://opensearch-cluster-master.os-stack.svc.cluster.local:9200ERR: An unexpected UnknownHostException occured: https://opensearch-cluster-master.os-stack.svc.cluster.local
Trace:
java.net.UnknownHostException: https://opensearch-cluster-master.os-stack.svc.cluster.local
    at java.base/sun.nio.ch.NioSocketImpl.connect(NioSocketImpl.java:567)
    at java.base/java.net.SocksSocketImpl.connect(SocksSocketImpl.java:327)
    at java.base/java.net.Socket.connect(Socket.java:751)
    at java.base/java.net.Socket.connect(Socket.java:686)
    at org.opensearch.security.tools.SecurityAdmin.execute(SecurityAdmin.java:521)
    at org.opensearch.security.tools.SecurityAdmin.main(SecurityAdmin.java:162)

When I run the curl -u admin:test -k --silent "https://localhost:9200" command from the shell of the opensearch-cluster-master pod, it indicates that the OpenSearch security is not initialized:

curl -u admin:test -k https://localhost:9200
OpenSearch Security not initialized.
[opensearch@opensearch-cluster-master-0 ~]$

If I try to run the securityadmin.sh command, it says:

ERR: "CN=opensearch-cluster-master-headless,OU=...,O=...,L=...,ST=...,C=..." is not an admin user
Seems you use a client certificate but this one is not registered as admin_dn

I recreated the certificates by slightly modifying the commands:

#!/bin/sh
# Root CA
openssl genrsa -out root-ca-key.pem 2048
openssl req -new -x509 -sha256 -key root-ca-key.pem -subj "/C=../ST=.../L=.../O=.../OU=.../CN=root" -out root-ca.pem -days 2048

# Generate key for nodes and admin
openssl genrsa -out node-key-temp.pem 2048
openssl pkcs8 -inform PEM -outform PEM -in node-key-temp.pem -topk8 -nocrypt -v1 PBE-SHA1-3DES -out node-key.pem

# Create CSR with SAN for admin and nodes
openssl req -new -key node-key.pem -subj "/C=../ST=.../L=.../O=.../OU=.../CN=opensearch-cluster-master-headless.os-stack.svc.cluster.local" -out node.csr

# Create SAN extension file
cat <<EOF > node.ext
subjectAltName = DNS:opensearch-cluster-master-headless.os-stack.svc.cluster.local, DNS:admin
EOF

# Sign certificate
openssl x509 -req -in node.csr -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial -sha256 -out node.pem -days 2048 -extfile node.ext

sh

After adding the subjectAltName, I achieved some progress. Now, when running the job you provided, I reach this point:


β”‚ Clustername: opensearch-cluster                                                                                      β”‚
β”‚ Clusterstate: YELLOW                                                                                                 β”‚
β”‚ Number of nodes: 1                                                                                                   β”‚
β”‚ Number of data nodes: 1                                                                                              β”‚
β”‚ ERR: An unexpected OpenSearchStatusException occurred: OpenSearch exception [type=cluster_manager_not_discovered_exception, reason=null]                    β”‚
β”‚ Trace:                                                                                                               β”‚
β”‚ OpenSearchStatusException[OpenSearch exception [type=cluster_manager_not_discovered_exception, reason=null]]                                              β”‚
β”‚     at org.opensearch.rest.BytesRestResponse.errorFromXContent(BytesRestResponse.java:210)                          β”‚
β”‚     at org.opensearch.client.RestHighLevelClient.parseEntity(RestHighLevelClient.java:2235)                          β”‚
β”‚     at org.opensearch.client.RestHighLevelClient.parseResponseException(RestHighLevelClient.java:2212)               β”‚
β”‚     at org.opensearch.client.RestHighLevelClient.internalPerformRequest(RestHighLevelClient.java:1931)               β”‚
β”‚     at org.opensearch.client.RestHighLevelClient.performRequest(RestHighLevelClient.java:1884)                       β”‚
β”‚     at org.opensearch.client.RestHighLevelClient.performRequestAndParseEntity(RestHighLevelClient.java:1852)         β”‚
β”‚     at org.opensearch.client.ClusterClient.health(ClusterClient.java:162)                                            β”‚
β”‚     at org.opensearch.security.tools.SecurityAdmin.execute(SecurityAdmin.java:821)                                   β”‚
β”‚     at org.opensearch.security.tools.SecurityAdmin.main(SecurityAdmin.java:162)                                      β”‚
β”‚     Suppressed: org.opensearch.client.ResponseException: method [GET], host [https://opensearch-cluster-master.os-stack.svc.cluster.local:9200], URI [/_cluster/health?master_timeout=30s&level=cluster&timeout=30s], status line [HTTP    β”‚
β”‚ {"error":{"root_cause":[{"type":"cluster_manager_not_discovered_exception","reason":null}],"type":"cluster_manager_not_discovered_exception","reason":null},"status":503}                           β”‚
β”‚         at org.opensearch.client.RestClient.convertResponse(RestClient.java:376)                                     β”‚
β”‚         at org.opensearch.client.RestClient.performRequest(RestClient.java:346)                                      β”‚
β”‚         at org.opensearch.client.RestClient.performRequest(RestClient.java:321)                                      β”‚
β”‚         at org.opensearch.client.RestHighLevelClient.internalPerformRequest(RestHighLevelClient.java:1918)           β”‚
β”‚         ... 5 more 

Do you have any advice?

1 Like

Ok, did you delete all resources created before you applied new certificates?

Also, I’m curious of the number of nodes. If you DON’T want to run OpenSearch Cluster by standalone mode (ie, only have ONE node which have both Master and Data role), at least 3 Master nodes are required to satisfy Quorum.
(For an election process to agree on an elected master node, both at startup and if the existing elected master fails)

Among three(at least) nodes, leader election starts. :sunny:

I have added the configuration node.name: opensearch-cluster-master-0, and now the job works.

**************************************************************************
** This tool will be deprecated in the next major release of OpenSearch **
** https://github.com/opensearch-project/security/issues/1755           **
**************************************************************************
Security Admin v7
Will connect to opensearch-cluster-master.os-stack.svc.cluster.local:9200 ... done
Connected as "CN=opensearch-cluster-master-headless.os-stack.svc.cluster.local,OU=Dev,O=Orbyta,L=Turin,ST=Italy,C=IT"
OpenSearch Version: 2.15.0
Diagnostic trace written to: /usr/share/opensearch/securityadmin_diag_trace_2024-Jul-29_09-13-42.txt
Contacting OpenSearch cluster 'opensearch' and waiting for YELLOW cluster state ...
Cluster name: opensearch-cluster
Cluster state: YELLOW
Number of nodes: 1
Number of data nodes: 1
.opendistro_security index already exists, so we do not need to create one.
Populate config from /usr/share/opensearch/config/opensearch-security/
Will update '/config' with /usr/share/opensearch/config/opensearch-security/config.yml
    SUCC: Configuration for 'config' created or updated
Will update '/roles' with /usr/share/opensearch/config/opensearch-security/roles.yml
    SUCC: Configuration for 'roles' created or updated
Will update '/rolesmapping' with /usr/share/opensearch/config/opensearch-security/roles_mapping.yml
    SUCC: Configuration for 'rolesmapping' created or updated
Will update '/internalusers' with /usr/share/opensearch/config/opensearch-security/internal_users.yml
    SUCC: Configuration for 'internalusers' created or updated
Will update '/actiongroups' with /usr/share/opensearch/config/opensearch-security/action_groups.yml
    SUCC: Configuration for 'actiongroups' created or updated
Will update '/tenants' with /usr/share/opensearch/config/opensearch-security/tenants.yml
    SUCC: Configuration for 'tenants' created or updated
Will update '/nodesdn' with /usr/share/opensearch/config/opensearch-security/nodes_dn.yml
    SUCC: Configuration for 'nodesdn' created or updated
Will update '/whitelist' with /usr/share/opensearch/config/opensearch-security/whitelist.yml
    SUCC: Configuration for 'whitelist' created or updated
SUCC: Expected 10 config types for node {"updated_config_types":["allowlist","tenants","rolesmapping","nodesdn","audit","roles","whitelist","actiongroups","config","internalusers"],"updated_config_size":10,"message":null} is 10.
Done with success                                                                                                                                                                                                                                                                                                     

Despite this, the pods opensearch-cluster-master-0 and opensearch-cluster-master-1 continue to give me the usual error:

opensearch [2024-07-29T09:18:40,178][DEBUG][o.o.t.TcpTransport       ] [opensearch-cluster-master-0] closed transport connection [316] to [{10.42.0.63:9300}{rn-Do74PSzi_vd6L8HIpcg}{opensearch-cluster-master-headless}{10.42.0.63:9300}] with age [0ms]
opensearch [2024-07-29T09:18:41,163][INFO ][o.o.s.c.ConfigurationRepository] [opensearch-cluster-master-0] Wait for cluster to be available ...
opensearch [2024-07-29T09:18:41,165][DEBUG][o.o.t.n.s.SecureNetty4Transport] [opensearch-cluster-master-0] Connection to opensearch-cluster-master-headless needs to be SSL, adding SSL handler to the client channel
opensearch [2024-07-29T09:18:41,174][DEBUG][o.o.t.TcpTransport       ] [opensearch-cluster-master-0] opened transport connection [317] to [{10.42.0.63:9300}{7kLi4qN9SkyO2uAA0AfPkQ}{opensearch-cluster-master-headless}{10.42.0.63:9300}] using channels [[Netty4TcpChannel{localAddress=/10.42.0.62:56596, remoteAddress=/10.42.0.63:9300}]]
opensearch [2024-07-29T09:18:41,174][DEBUG][o.o.t.TransportService   ] [opensearch-cluster-master-0] Action: internal:transport/handshake
opensearch [2024-07-29T09:18:41,177][WARN ][o.o.d.HandshakingTransportAddressConnector] [opensearch-cluster-master-0] handshake failed for [connectToRemoteMasterNode[10.42.0.63:9300]]
opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.0.63:9300][internal:transport/handshake]
opensearch Caused by: org.opensearch.OpenSearchException: Transport client authentication no longer supported.
opensearch     at org.opensearch.security.ssl.util.ExceptionUtils.createTransportClientNoLongerSupportedException(ExceptionUtils.java:80) ~[?:?]
opensearch     at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:294) ~[?:?]
opensearch     at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:155) ~[?:?]
opensearch     at org.opensearch.security.OpenSearchSecurityPlugin$6$1.messageReceived(OpenSearchSecurityPlugin.java:866) ~[?:?]
opensearch     at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:114) ~[?:?]
opensearch     at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:43) ~[?:?]
opensearch     at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:108) ~[opensearch-2.15.0.jar:2.15.0]
opensearch     at org.opensearch.transport.NativeMessageHandler.handleRequest(NativeMessageHandler.java:278) ~[opensearch-2.15.0.jar:2.15.0]
opensearch     at org.opensearch.transport.NativeMessageHandler.handleMessage(NativeMessageHandler.java:146) ~[opensearch-2.15.0.jar:2.15.0]
opensearch     at org.opensearch.transport.NativeMessageHandler.messageReceived(NativeMessageHandler.java:126) ~[opensearch-2.15.0.jar:2.15.0]
opensearch     at org.opensearch.transport.InboundHandler.messageReceivedFromPipeline(InboundHandler.java:121) ~[opensearch-2.15.0.jar:2.15.0]
opensearch     at org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:113) ~[opensearch-2.15.0.jar:2.15.0]
opensearch     at org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:800) ~[opensearch-2.15.0.jar:2.15.0]
opensearch     at org.opensearch.transport.nativeprotocol.NativeInboundBytesHandler.forwardFragments(NativeInboundBytesHandler.java:157) ~[opensearch-2.15.0.jar:2.15.0]
opensearch     at org.opensearch.transport.nativeprotocol.NativeInboundBytesHandler.doHandleBytes(NativeInboundBytesHandler.java:94) ~[opensearch-2.15.0.jar:2.15.0]
opensearch     at org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:143) ~[opensearch-2.15.0.jar:2.15.0]
opensearch     at org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:119) ~[opensearch-2.15.0.jar:2.15.0]
opensearch     at org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:95) ~[?:?]
opensearch     at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[?:?]
opensearch     at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[?:?]
opensearch     at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[?:?]
opensearch     at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) ~[?:?]
opensearch     at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[?:?]
opensearch     at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[?:?]
opensearch     at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[?:?]
opensearch     at io.netty.handler.codec

.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) ~[?:?]
opensearch     at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[?:?]
opensearch     at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[?:?]
opensearch     at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[?:?]
opensearch     at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1475) ~[?:?]
opensearch     at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1338) ~[?:?]
opensearch     at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1387) ~[?:?]
opensearch     at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[?:?]
opensearch     at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[?:?]
opensearch     at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[?:?]
opensearch     at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[?:?]
opensearch     at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[?:?]
opensearch     at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[?:?]
opensearch     at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1407) ~[?:?]
opensearch     at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[?:?]
opensearch     at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[?:?]
opensearch     at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:918) ~[?:?]
opensearch     at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[?:?]
opensearch     at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[?:?]
opensearch     at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:689) ~[?:?]
opensearch     at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:652) ~[?:?]
opensearch     at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[?:?]
opensearch     at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) ~[?:?]
opensearch     at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]
opensearch     at java.lang.Thread.run(Thread.java:1583) [?:?]
opensearch [2024-07-29T09:18:41,179][DEBUG][o.o.t.TcpTransport       ] [opensearch-cluster-master-0] closed transport connection [317] to [{10.42.0.63:9300}{7kLi4qN9SkyO2uAA0AfPkQ}{opensearch-cluster-master-headless}{10.42.0.63:9300}] with age [0ms]

Anyway i have 3 nodes (opensearch-cluster-master-0,opensearch-cluster-master-1,opensearch-cluster-master-2) but probably I assigned every roles to the same node.

node.name: opensearch-cluster-master-0 
masterService: "opensearch-cluster-master"
roles:
  - cluster_manager
  - ingest
  - data
  - remote_cluster_client

At the very beggining in the post, only two nodes are specified to discover each other.

discovery.seed_hosts: ["opensearch-cluster-master-0", "opensearch-cluster-master-1"]

Yeah, I’m sorry. I added the third one after you told me that I need 3 nodes, but after that, the job stopped working.

This is what happens in the job when i add the third node "

β”‚ ** This tool will be deprecated in the next major release of OpenSearch **                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     β”‚
β”‚ ** https://github.com/opensearch-project/security/issues/1755           **                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     β”‚
β”‚ **************************************************************************                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     β”‚
β”‚ Security Admin v7                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              β”‚
β”‚ Will connect to opensearch-cluster-master.os-stack.svc.cluster.local:9200 ... done                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             β”‚
β”‚ Connected as "CN=opensearch-cluster-master-headless.os-stack.svc.cluster.local,OU=Dev,O=Orbyta,L=Turin,ST=Italy,C=IT"                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          β”‚
β”‚ OpenSearch Version: 2.15.0                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     β”‚
β”‚ Diagnostic trace written to: /usr/share/opensearch/securityadmin_diag_trace_2024-Jul-29_09-43-06.txt                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           β”‚
β”‚ Contacting opensearch cluster 'opensearch' and wait for YELLOW clusterstate ...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                β”‚


β”‚ Cannot retrieve cluster state due to: 30,000 milliseconds timeout on connection http-outgoing-13 [ACTIVE]. This is not an error, will keep on trying ...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       β”‚
β”‚   Root cause: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-13 [ACTIVE] (java.net.SocketTimeoutException/java.net.SocketTimeoutException)                                                                                                                                                                                                                                                                                                                                                                                                                                                           β”‚
β”‚    * Try running securityadmin.sh with -icl (but no -cl) and -nhnv (If that works you need to check your clustername as well as hostnames in your TLS certificates)                                                                                                                                                                                                                                                                                                                                                                                                                                                                            β”‚
β”‚    * Make sure that your keystore or PEM certificate is a client certificate (not a node certificate) and configured properly in opensearch.yml                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                β”‚
β”‚    * If this is not working, try running securityadmin.sh with --diagnose and see diagnose trace log file)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     β”‚
β”‚    * Add --accept-red-cluster to allow securityadmin to operate on a red cluster.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              β”‚
β”‚ Cannot retrieve cluster state due to: 30,000 milliseconds timeout on connection http-outgoing-14 [ACTIVE]. This is not an error, will keep on trying ...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       β”‚
β”‚   Root cause: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-14 [ACTIVE] (java.net.SocketTimeoutException/java.net.SocketTimeoutException)                                                                                                                                                                                                                                                                                                                                                                                                                                                           β”‚
β”‚    * Try running securityadmin.sh with -icl (but no -cl) and -nhnv (If that works you need to check your clustername as well as hostnames in your TLS certificates)                                                                                                                                                                                                                                                                                                                                                                                                                                                                            β”‚
β”‚    * Make sure that your keystore or PEM certificate is a client certificate (not a node certificate) and configured properly in opensearch.yml                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                β”‚
β”‚    * If this is not working, try running securityadmin.sh with --diagnose and see diagnose trace log file)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     β”‚
β”‚    * Add --accept-red-cluster to allow securityadmin to operate on a red cluster.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              β”‚
β”‚ Cannot retrieve cluster state due to: 30,000 milliseconds timeout on connection http-outgoing-15 [ACTIVE]. This is not an error, will keep on trying ...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       β”‚
β”‚   Root cause: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-15 [ACTIVE] (java.net.SocketTimeoutException/java.net.SocketTimeoutException)                                                                                                                                                                                                                                                                                                                                                                                                                                                           β”‚
β”‚    * Try running securityadmin.sh with -icl (but no -cl) and -nhnv (If that works you need to check your clustername as well as hostnames in your TLS certificates)                                                                                                                                                                                                                                                                                                                                                                                                                                                                            β”‚
β”‚    * Make sure that your keystore or PEM certificate is a client certificate (not a node certificate) and configured properly in opensearch.yml                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                β”‚
β”‚    * If this is not working, try running securityadmin.sh with --diagnose and see diagnose trace log file)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     β”‚
β”‚    * Add --accept-red-cluster to allow securityadmin to operate on a red cluster.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              β”‚
β”‚ Cannot retrieve cluster state due to: 30,000 milliseconds timeout on connection http-outgoing-16 [ACTIVE]. This is not an error, will keep on trying ...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       β”‚
β”‚   Root cause: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-16 [ACTIVE] (java.net.SocketTimeoutException/java.net.SocketTimeoutException)                                                                                                                                                                                                                                                                                                                                                                                                                                                           β”‚
β”‚    * Try running securityadmin.sh with -icl (but no -cl) and -nhnv (If that works you need to check your clustername as well as hostnames in your TLS certificates)                                                                                                                                                                                                                                                                                                                                                                                                                                                                            β”‚
β”‚    * Make sure that your keystore or PEM certificate is a client certificate (not a node certificate) and configured properly in opensearch.yml                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                β”‚
β”‚    * If this is not working, try running securityadmin.sh with --diagnose and see diagnose trace log file)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     β”‚
β”‚    * Add --accept-red-cluster to allow securityadmin to operate on a red cluster.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              β”‚
β”‚ Cannot retrieve cluster state due to: 30,000 milliseconds timeout on connection http-outgoing-17 [ACTIVE]. This is not an error, will keep on trying ...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       β”‚
β”‚   Root cause: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-17 [ACTIVE] (java.net.SocketTimeoutException/java.net.SocketTimeoutException)                                                                                                                                                                                                                                                                                                                                                                                                                                                           β”‚
β”‚    * Try running securityadmin.sh with -icl (but no -cl) and -nhnv (If that works you need to check your clustername as well as hostnames in your TLS certificates)                                                                                                                                                                                                                                                                                                                                                                                                                                                                            β”‚
β”‚    * Make sure that your keystore or PEM certificate is a client certificate (not a node certificate) and configured properly in opensearch.yml                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                β”‚
β”‚    * If this is not working, try running securityadmin.sh with --diagnose and see diagnose trace log file)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     β”‚
β”‚    * Add --accept-red-cluster to allow securityadmin to operate on a red cluster.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              β”‚
β”‚ Cannot retrieve cluster state due to: 30,000 milliseconds timeout on connection http-outgoing-18 [ACTIVE]. This is not an error, will keep on trying ...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       β”‚
β”‚   Root cause: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-18 [ACTIVE] (java.net.SocketTimeoutException/java.net.SocketTimeoutException)                                                                                                                                                                                                                                                                                                                                                                                                                                                           β”‚
β”‚    * Try running securityadmin.sh with -icl (but no -cl) and -nhnv (If that works you need to check your clustername as well as hostnames in your TLS certificates)                                                                                                                                                                                                                                                                                                                                                                                                                                                                            β”‚
β”‚    * Make sure that your keystore or PEM certificate is a client certificate (not a node certificate) and configured properly in opensearch.yml                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                β”‚
β”‚    * If this is not working, try running securityadmin.sh with --diagnose and see diagnose trace log file)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     β”‚
β”‚    * Add --accept-red-cluster to allow securityadmin to operate on a red cluster.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              β”‚
β”‚ Clustername: opensearch-cluster                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                β”‚
β”‚ Clusterstate: YELLOW                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           β”‚
β”‚ Number of nodes: 1                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             β”‚
β”‚ Number of data nodes: 1                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        β”‚
β”‚ Unable to get index because return code was 503                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                β”‚
β”‚ **************************************************************************                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     β”‚
β”‚ ** This tool will be deprecated in the next major release of OpenSearch **                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     β”‚
β”‚ ** https://github.com/opensearch-project/security/issues/1755           **                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     β”‚
β”‚ **************************************************************************                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     β”‚
β”‚ Security Admin v7                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              β”‚
β”‚ Will connect to opensearch-cluster-master.os-stack.svc.cluster.local:9200 ... done                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             β”‚
β”‚ Connected as "CN=opensearch-cluster-master-headless.os-stack.svc.cluster.local,OU=Dev,O=Orbyta,L=Turin,ST=Italy,C=IT"                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          β”‚
β”‚ OpenSearch Version: 2.15.0                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     β”‚
β”‚ Diagnostic trace written to: /usr/share/opensearch/securityadmin_diag_trace_2024-Jul-29_09-50-32.txt                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           β”‚
β”‚ Contacting opensearch cluster 'opensearch' and wait for YELLOW clusterstate ...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                β”‚
β”‚                                                                               ```

Can you access any pod(among three nodes) and run curl https://localhost:9200 -k -u username:password??

The job should be stopped after leader election finishes successfully.

[opensearch@opensearch-cluster-master-0 ~]$ curl https://localhost:9200 -k -u admin:test
OpenSearch Security not initialized.[opensearch@opensearch-cluster-master-0 ~]$ curl https://localhost:9200 -k -u admin:test
OpenSearch Security not initialized.[opensearch@opensearch-cluster-master-0 ~]$ curl https://localhost:9200 -k -u admin:test
OpenSearch Security not initialized.[opensearch@opensearch-cluster-master-0 ~]$ curl https://localhost:9200 -k -u admin:test
OpenSearch Security not initialized.[opensearch@opensearch-cluster-master-0 ~]$

If i try to run the same command but with opensearch-cluster-master (Kubernetes service) it works

OpenSearch Security not initialized.[opensearch@opensearch-cluster-master-0 ~]$ curl https://opensearch-cluster-master:9200 -k -u admin:test
{
  "name" : "opensearch-cluster-master-1",
  "cluster_name" : "opensearch-cluster",
  "cluster_uuid" : "mD5n-tasQiiwxgMk4-Xywg",
  "version" : {
    "distribution" : "opensearch",
    "number" : "2.15.0",
    "build_type" : "tar",
    "build_hash" : "61dbcd0795c9bfe9b81e5762175414bc38bbcadf",
    "build_date" : "2024-06-20T03:26:49.193630411Z",
    "build_snapshot" : false,
    "lucene_version" : "9.10.0",
    "minimum_wire_compatibility_version" : "7.10.0",
    "minimum_index_compatibility_version" : "7.0.0"
  },
  "tagline" : "The OpenSearch Project: https://opensearch.org/"
}```
[opensearch@opensearch-cluster-master-0 ~]$

I noticed that the following cURL command always returns results from opensearch-cluster-master-1.

curl https://opensearch-cluster-master:9200 -k -u admin:test

When I run the command
curl -k -u admin:test https://localhost:9200
on the master-1 node, it works correctly.
It appears that OpenSearch security has been initialized on only one of the three nodes.
That explains why the nodes couldn’t communicate properly.

Here’s my updated configuration, incorporating your suggestions along with a few additional changes. At this point, I have one of the three nodes (opensearch-cluster-master-1) with the OpenSearch security plugin correctly configured, while the other two nodes (opensearch-cluster-master-0 and opensearch-cluster-master-2) have the plugin not initialized.
I believe this is why the nodes can’t communicate with each other.

---
clusterName: "opensearch-cluster"
nodeGroup: "master"
discovery.seed_hosts: ["opensearch-cluster-master-0", "opensearch-cluster-master-1", "opensearch-cluster-master-2"]
singleNode: false
masterService: "opensearch-cluster-master"
node.name: opensearch-cluster-master-0 
cluster.initial_cluster_manager_nodes: ["opensearch-cluster-master-0"]
roles:
  - cluster_manager
  - ingest
  - data
  - remote_cluster_client
replicas: 3

global:
  dockerRegistry: ""

opensearchHome: /usr/share/opensearch

config:
  opensearch.yml: |
    cluster.name: opensearch-cluster
    http.host: "0.0.0.0"
    plugins.query.datasources.encryption.masterkey: "1e5a53e5e9addb4cb96448b3"
    # Configurazione SSL/TLS per il trasporto
    plugins.security.ssl.transport.enabled: true
    plugins.security.ssl.transport.pemcert_filepath: /usr/share/opensearch/config/certs/node.pem
    plugins.security.ssl.transport.pemkey_filepath: /usr/share/opensearch/config/certs/node-key.pem
    plugins.security.ssl.transport.pemtrustedcas_filepath: /usr/share/opensearch/config/certs/root-ca.pem
    plugins.security.ssl.transport.enforce_hostname_verification: false

    # Configurazione SSL/TLS per HTTP
    plugins.security.ssl.http.enabled: true
    plugins.security.ssl.http.pemcert_filepath: /usr/share/opensearch/config/certs/node.pem
    plugins.security.ssl.http.pemkey_filepath: /usr/share/opensearch/config/certs/node-key.pem
    plugins.security.ssl.http.pemtrustedcas_filepath: /usr/share/opensearch/config/certs/root-ca.pem
    # # Configurazione dei DN autorizzati
    plugins.security.nodes_dn:
      - 'CN=admin,OU=...,O=...,L=...,ST=...,C=...'
    plugins.security.nodes_dn_dynamic_config_enabled: true  
    # Altre impostazioni di sicurezza
    plugins.security.allow_unsafe_democertificates: false
    plugins.security.allow_default_init_securityindex: true
    plugins.security.authcz.admin_dn:
      - 'CN=opensearch-cluster-master-headless.os-stack.svc.cluster.local,OU=...,O=...,L=...,ST=...,C=IT'
    plugins.security.audit.type: internal_opensearch
    plugins.security.enable_snapshot_restore_privilege: true
    plugins.security.check_snapshot_restore_write_privileges: true
    plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"]
    plugins.security.system_indices.enabled: true
    plugins.security.system_indices.indices: [ ".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opendistro-notifications-*", ".opendistro-notebooks", ".opendistro-asynchronous-search-response*"]
    logger.org.opensearch.transport: DEBUG
    logger.org.opensearch.security: DEBUG
extraEnvs: 
  - name: "OPENSEARCH_INITIAL_ADMIN_PASSWORD"
    value: "..."
  - name: DISABLE_INSTALL_DEMO_CONFIG
    value: "true"



envFrom: []

secretMounts: 
  - name: open-certificates
    secretName: open-certificates
    path: /usr/share/opensearch/config/certs/

hostAliases: []

image:
  repository: "opensearchproject/opensearch"
  tag: "2.15.0"
  pullPolicy: "IfNotPresent"

podAnnotations: {}

openSearchAnnotations: {}

labels: {}

opensearchJavaOpts: "-Xmx512M -Xms512M"

resources:
  requests:
    cpu: "1000m"
    memory: "100Mi"

initResources: {}

sidecarResources: {}

networkHost: "0.0.0.0"

rbac:
  create: false
  serviceAccountAnnotations: {}
  serviceAccountName: ""
  automountServiceAccountToken: false

podSecurityPolicy:
  create: false
  name: ""
  spec:
    privileged: true
    fsGroup:
      rule: RunAsAny
    runAsUser:
      rule: RunAsAny
    seLinux:
      rule: RunAsAny
    supplementalGroups:
      rule: RunAsAny
    volumes:
      - secret
      - configMap
      - persistentVolumeClaim
      - emptyDir

persistence:
  enabled: true
  enableInitChown: true
  labels:
    enabled: false
  accessModes:
    - ReadWriteOnce
  size: 8Gi
  annotations: {}

# extraVolumes:
#   - name: securityconfig
#     secret:
#       secretName: securityconfig-secret
#   - name: admin-credentials
#     secret:
#       secretName: admin-credentials-secr

# extraVolumeMounts:
#   - name: securityconfig
#     mountPath: /usr/share/opensearch/config/opensearch-security
#     readOnly: true
#   - name: admin-credentials
#     mountPath: /usr/share/opensearch/config/admin
#     readOnly: true

extraContainers: []

extraInitContainers: []

priorityClassName: ""

antiAffinityTopologyKey: "kubernetes.io/hostname"
antiAffinity: "soft"
customAntiAffinity: {}

nodeAffinity: {}

podAffinity: {}

topologySpreadConstraints: []

podManagementPolicy: "Parallel"
enableServiceLinks: true

protocol: https
httpPort: 9200
transportPort: 9300
metricsPort: 9600
httpHostPort: ""
transportHostPort: ""

service:
  labels: {}
  labelsHeadless: {}
  headless:
    annotations: {}
  type: ClusterIP
  nodePort: ""
  annotations: {}
  httpPortName: http
  transportPortName: transport
  metricsPortName: metrics
  loadBalancerIP: ""
  loadBalancerSourceRanges: []
  externalTrafficPolicy: ""

updateStrategy: RollingUpdate
maxUnavailable: 1

podSecurityContext:
  fsGroup: 1000
  runAsUser: 1000

securityContext:
  capabilities:
    drop:
      - ALL
  runAsNonRoot: true
  runAsUser: 1000

securityConfig:
  enabled: true
  path: "/usr/share/opensearch/config/opensearch-security"
  config:
    data: 
      action_groups.yml: |-
         _meta:
           type: "actiongroups"
           config_version: 2
      internal_users.yml: |-
        _meta:
          type: "internalusers"
          config_version: 2
        admin:
          hash: "$2y$12$tS0wrbNssQpVjOXDPrzqdO5phJC/Fmb9fNKSdJ9P2voGK.LNIqLxG"
          reserved: true
          backend_roles:
          - "admin"
          description: "Demo admin user"
        dashboarduser:
          hash: "$2a$12$4AcgAt3xwOWadA5s5blL6ev39OXDNhmOesEoo33eZtrq2N0YrU3H."
          reserved: true
          description: "Demo OpenSearch Dashboards user"
      nodes_dn.yml: |-
        _meta:
          type: "nodesdn"
          config_version: 2
      whitelist.yml: |-
        _meta:
          type: "whitelist"
          config_version: 2
      tenants.yml: |-
        _meta:
          type: "tenants"
          config_version: 2
      roles_mapping.yml: |-
        _meta:
          type: "rolesmapping"
          config_version: 2
        all_access:
          reserved: false
          backend_roles:
          - "admin"
          description: "Maps admin to all_access"
        own_index:
          reserved: false
          users:
          - "*"
          description: "Allow full access to an index named like the username"
        readall:
          reserved: false
          backend_roles:
          - "readall"
        manage_snapshots:
          reserved: false
          backend_roles:
          - "snapshotrestore"
        dashboard_server:
          reserved: true
          users:
          - "dashboarduser"
      roles.yml: |-
        _meta:
          type: "roles"
          config_version: 2
        dashboard_read_only:
          reserved: true
        security_rest_api_access:
          reserved: true
        # Allows users to view monitors, destinations and alerts
        alerting_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/alerting/alerts/get'
            - 'cluster:admin/opendistro/alerting/destination/get'
            - 'cluster:admin/opendistro/alerting/monitor/get'
            - 'cluster:admin/opendistro/alerting/monitor/search'
        # Allows users to view and acknowledge alerts
        alerting_ack_alerts:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/alerting/alerts/*'
        # Allows users to use all alerting functionality
        alerting_full_access:
          reserved: true
          cluster_permissions:
            - 'cluster_monitor'
            - 'cluster:admin/opendistro/alerting/*'
          index_permissions:
            - index_patterns:
                - '*'
              allowed_actions:
                - 'indices_monitor'
                - 'indices:admin/aliases/get'
                - 'indices:admin/mappings/get'
        # Allow users to read Anomaly Detection detectors and results
        anomaly_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/ad/detector/info'
            - 'cluster:admin/opendistro/ad/detector/search'
            - 'cluster:admin/opendistro/ad/detectors/get'
            - 'cluster:admin/opendistro/ad/result/search'
            - 'cluster:admin/opendistro/ad/tasks/search'
            - 'cluster:admin/opendistro/ad/detector/validate'
            - 'cluster:admin/opendistro/ad/result/topAnomalies'
        # Allows users to use all Anomaly Detection functionality
        anomaly_full_access:
          reserved: true
          cluster_permissions:
            - 'cluster_monitor'
            - 'cluster:admin/opendistro/ad/*'
          index_permissions:
            - index_patterns:
                - '*'
              allowed_actions:
                - 'indices_monitor'
                - 'indices:admin/aliases/get'
                - 'indices:admin/mappings/get'
        # Allows users to read Notebooks
        notebooks_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/notebooks/list'
            - 'cluster:admin/opendistro/notebooks/get'
        # Allows users to all Notebooks functionality
        notebooks_full_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/notebooks/create'
            - 'cluster:admin/opendistro/notebooks/update'
            - 'cluster:admin/opendistro/notebooks/delete'
            - 'cluster:admin/opendistro/notebooks/get'
            - 'cluster:admin/opendistro/notebooks/list'
        # Allows users to read observability objects
        observability_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opensearch/observability/get'
        # Allows users to all Observability functionality
        observability_full_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opensearch/observability/create'
            - 'cluster:admin/opensearch/observability/update'
            - 'cluster:admin/opensearch/observability/delete'
            - 'cluster:admin/opensearch/observability/get'
        # Allows users to read and download Reports
        reports_instances_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/reports/instance/list'
            - 'cluster:admin/opendistro/reports/instance/get'
            - 'cluster:admin/opendistro/reports/menu/download'
        # Allows users to read and download Reports and Report-definitions
        reports_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/reports/definition/get'
            - 'cluster:admin/opendistro/reports/definition/list'
            - 'cluster:admin/opendistro/reports/instance/list'
            - 'cluster:admin/opendistro/reports/instance/get'
            - 'cluster:admin/opendistro/reports/menu/download'
        # Allows users to all Reports functionality
        reports_full_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/reports/definition/create'
            - 'cluster:admin/opendistro/reports/definition/update'
            - 'cluster:admin/opendistro/reports/definition/on_demand'
            - 'cluster:admin/opendistro/reports/definition/delete'
            - 'cluster:admin/opendistro/reports/definition/get'
            - 'cluster:admin/opendistro/reports/definition/list'
            - 'cluster:admin/opendistro/reports/instance/list'
            - 'cluster:admin/opendistro/reports/instance/get'
            - 'cluster:admin/opendistro/reports/menu/download'
        # Allows users to use all asynchronous-search functionality
        asynchronous_search_full_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/asynchronous_search/*'
          index_permissions:
            - index_patterns:
                - '*'
              allowed_actions:
                - 'indices:data/read/search*'
        # Allows users to read stored asynchronous-search results
        asynchronous_search_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/asynchronous_search/get'
        # Allows user to use all index_management actions - ism policies, rollups, transforms
        index_management_full_access:
          reserved: true
          cluster_permissions:
            - "cluster:admin/opendistro/ism/*"
            - "cluster:admin/opendistro/rollup/*"
            - "cluster:admin/opendistro/transform/*"
          index_permissions:
            - index_patterns:
                - '*'
              allowed_actions:
                - 'indices:admin/opensearch/ism/*'
        # Allows users to use all cross cluster replication functionality at leader cluster
        cross_cluster_replication_leader_full_access:
          reserved: true
          index_permissions:
            - index_patterns:
                - '*'
              allowed_actions:
                - "indices:admin/plugins/replication/index/setup/validate"
                - "indices:data/read/plugins/replication/changes"
                - "indices:data/read/plugins/replication/file_chunk"
        # Allows users to use all cross cluster replication functionality at follower cluster
        cross_cluster_replication_follower_full_access:
          reserved: true
          cluster_permissions:
            - "cluster:admin/plugins/replication/autofollow/update"
          index_permissions:
            - index_patterns:
                - '*'
              allowed_actions:
                - "indices:admin/plugins/replication/index/setup/validate"
                - "indices:data/write/plugins/replication/changes"
                - "indices:admin/plugins/replication/index/start"
                - "indices:admin/plugins/replication/index/pause"
                - "indices:admin/plugins/replication/index/resume"
                - "indices:admin/plugins/replication/index/stop"
                - "indices:admin/plugins/replication/index/update"
                - "indices:admin/plugins/replication/index/status_check"
      config.yml: |-
        _meta:
          type: "config"
          config_version: 2
        config:
          dynamic:
            authc:
              basic_internal_auth_domain:
                http_enabled: true
                transport_enabled: true
                order: 0
                http_authenticator:
                  type: basic
                  challenge: false
                authentication_backend:
                  type: internal
              openid_auth_domain:
                http_enabled: true
                transport_enabled: true
                order: 1
                http_authenticator:
                  type: openid
                  challenge: false
                  config:
                    subject_key: admin
                    openid_connect_url: https://keycloak.example.com/realms/master/.well-known/openid-configuration
                authentication_backend:
                  type: noop

terminationGracePeriod: 120

sysctlVmMaxMapCount: 262144

startupProbe:
  tcpSocket:
    port: 9200
  initialDelaySeconds: 5
  periodSeconds: 10
  timeoutSeconds: 3
  failureThreshold: 30

livenessProbe: {}

readinessProbe:
  tcpSocket:
    port: 9200
  periodSeconds: 5
  timeoutSeconds: 3
  failureThreshold: 3

schedulerName: ""

imagePullSecrets: []
nodeSelector: {}
tolerations: []

ingress:
  enabled: false
  ingressClassName: nginx
  path: /
  hosts:
    - opensearch.example.local
  tls:
    - hosts:
      - opensearch.example.local
  annotations: 
    nginx.ingress.kubernetes.io/proxy-connect-timeout: "60s"
    nginx.ingress.kubernetes.io/proxy-read-timeout: "600s"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "600s"
    nginx.ingress.kubernetes.io/proxy-buffering: "on"
    nginx.ingress.kubernetes.io/proxy-buffer-size: "64k"

nameOverride: ""
fullnameOverride: ""

masterTerminationFix: false

opensearchLifecycle: {}

lifecycle: {}

keystore: []

networkPolicy:
  create: false
  http:
    enabled: false

sysctl:
  enabled: false

sysctlInit:
  enabled: false

plugins:
  enabled: true
  installList: []

extraObjects: []

That sounds correct. Is there a reason why you would have the security plugin enabled on the master node but not the other nodes?

best,
mj

I would like the plugin to be enabled on all nodes, and I’m not sure why it is behaving this way.
From the configuration, there is no indication of which nodes it should be enabled on. Additionally, the only node it is enabled on is master-1, which is never mentioned in the configuration.
Is there a specific section where it should be enabled for the other nodes?
Do you have any advice?