Multiple exceptions related to plugins.security

Versions (relevant - OpenSearch/Dashboard/Server OS/Browser):

helm repo add opensearch helm-charts | ☸ A community repository for Helm Charts of OpenSearch Project.
helm repo update
helm search repo opensearch
helm install my-deployment opensearch/opensearch --values values.yaml

Describe the issue:

After deploying in k8s using helm, getting security related exceptions.

opensearch-cluster-master-0 status is orange
opensearch-cluster-master-1 status is green
opensearch-cluster-master-2 status is orange

Configuration:

values.yaml =

---
clusterName: "opensearch-cluster"
nodeGroup: "master"

# If discovery.type in the opensearch configuration is set to "single-node",
# this should be set to "true"
# If "true", replicas will be forced to 1
singleNode: false

# The service that non master groups will try to connect to when joining the cluster
# This should be set to clusterName + "-" + nodeGroup for your master group
masterService: "opensearch-cluster-master"

# OpenSearch roles that will be applied to this nodeGroup
# These will be set as environment variable "node.roles". E.g. node.roles=master,ingest,data,remote_cluster_client
roles:
  - master
  - ingest
  - data
  - remote_cluster_client

replicas: 3

# if not set, falls back to parsing .Values.imageTag, then .Chart.appVersion.
majorVersion: ""

global:
  # Set if you want to change the default docker registry, e.g. a private one.
  dockerRegistry: ""

# Allows you to add any config files in {{ .Values.opensearchHome }}/config
opensearchHome: /usr/share/opensearch

# such as opensearch.yml and log4j2.properties
config:
  # Values must be YAML literal style scalar / YAML multiline string.
  # <filename>: |
  #   <formatted-value(s)>
  # log4j2.properties: |
  #   status = error
  #
  #   appender.console.type = Console
  #   appender.console.name = console
  #   appender.console.layout.type = PatternLayout
  #   appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n
  #
  #   rootLogger.level = info
  #   rootLogger.appenderRef.console.ref = console
  opensearch.yml: |
    cluster.name: opensearch-cluster

    # Bind to all interfaces because we don't know what IP address Docker will assign to us.
    network.host: 0.0.0.0

    # Setting network.host to a non-loopback address enables the annoying bootstrap checks. "Single-node" mode disables them again.
    # Implicitly done if ".singleNode" is set to "true".
    # discovery.type: single-node

    # Start OpenSearch Security Demo Configuration
    # WARNING: revise all the lines below before you go into production
    plugins:
      security:
        ssl:
          transport:
            pemcert_filepath: esnode.pem
            pemkey_filepath: esnode-key.pem
            pemtrustedcas_filepath: root-ca.pem
            enforce_hostname_verification: false
          http:
            enabled: true
            pemcert_filepath: esnode.pem
            pemkey_filepath: esnode-key.pem
            pemtrustedcas_filepath: root-ca.pem
        allow_unsafe_democertificates: true
        allow_default_init_securityindex: true
        authcz:
          admin_dn:
            - CN=kirk,OU=client,O=client,L=test,C=de
        audit.type: internal_opensearch
        enable_snapshot_restore_privilege: true
        check_snapshot_restore_write_privileges: true
        restapi:
          roles_enabled: ["all_access", "security_rest_api_access"]
        system_indices:
          enabled: true
          indices:
            [
              ".opendistro-alerting-config",
              ".opendistro-alerting-alert*",
              ".opendistro-anomaly-results*",
              ".opendistro-anomaly-detector*",
              ".opendistro-anomaly-checkpoints",
              ".opendistro-anomaly-detection-state",
              ".opendistro-reports-*",
              ".opendistro-notifications-*",
              ".opendistro-notebooks",
              ".opendistro-asynchronous-search-response*",
            ]
    ######## End OpenSearch Security Demo Configuration ########
  # log4j2.properties:

# Extra environment variables to append to this nodeGroup
# This will be appended to the current 'env:' key. You can use any of the kubernetes env
# syntax here
extraEnvs:
#  - name: MY_ENVIRONMENT_VAR
#    value: the_value_goes_here
# Chart version 2.18.0 and App Version OpenSearch 2.12.0 onwards a custom strong password needs to be provided in order to setup demo admin user.
# Cluster will not spin-up without this unless demo config install is disabled.
  - name: OPENSEARCH_INITIAL_ADMIN_PASSWORD
    value: "aHo1@pizZ#"
# Allows you to load environment variables from kubernetes secret or config map
envFrom: []
# - secretRef:
#     name: env-secret
# - configMapRef:
#     name: config-map

# A list of secrets and their paths to mount inside the pod
# This is useful for mounting certificates for security and for mounting
# the X-Pack license
secretMounts: []

hostAliases: []
# - ip: "127.0.0.1"
#   hostnames:
#   - "foo.local"
#   - "bar.local"

image:
  repository: "opensearchproject/opensearch"
  # override image tag, which is .Chart.AppVersion by default
  tag: ""
  pullPolicy: "IfNotPresent"

podAnnotations: {}
  # iam.amazonaws.com/role: es-cluster

# OpenSearch Statefulset annotations
openSearchAnnotations: {}

# additionals labels
labels: {}

opensearchJavaOpts: "-Xmx512M -Xms512M"

resources:
  requests:
    cpu: "1000m"
    memory: "100Mi"

initResources: {}
#  limits:
#     cpu: "25m"
#     memory: "128Mi"
#  requests:
#     cpu: "25m"
#     memory: "128Mi"

sidecarResources: {}
#   limits:
#     cpu: "25m"
#     memory: "128Mi"
#   requests:
#     cpu: "25m"
#     memory: "128Mi"

networkHost: "0.0.0.0"

rbac:
  create: false
  serviceAccountAnnotations: {}
  serviceAccountName: ""
  # Controls whether or not the Service Account token is automatically mounted to /var/run/secrets/kubernetes.io/serviceaccount
  automountServiceAccountToken: false

podSecurityPolicy:
  create: false
  name: ""
  spec:
    privileged: true
    fsGroup:
      rule: RunAsAny
    runAsUser:
      rule: RunAsAny
    seLinux:
      rule: RunAsAny
    supplementalGroups:
      rule: RunAsAny
    volumes:
      - secret
      - configMap
      - persistentVolumeClaim
      - emptyDir

persistence:
  enabled: true
  # Set to false to disable the `fsgroup-volume` initContainer that will update permissions on the persistent disk.
  enableInitChown: true
  # override image, which is busybox by default
  # image: busybox
  # override image tag, which is latest by default
  # imageTag:
  labels:
    # Add default labels for the volumeClaimTemplate of the StatefulSet
    enabled: false
  # OpenSearch Persistent Volume Storage Class
  # If defined, storageClassName: <storageClass>
  # If set to "-", storageClassName: "", which disables dynamic provisioning
  # If undefined (the default) or set to null, no storageClassName spec is
  #   set, choosing the default provisioner.  (gp2 on AWS, standard on
  #   GKE, AWS & OpenStack)
  #
  # storageClass: "-"
  accessModes:
    - ReadWriteOnce
  size: 8Gi
  annotations: {}

extraVolumes: []
  # - name: extras
  #   emptyDir: {}

extraVolumeMounts: []
  # - name: extras
  #   mountPath: /usr/share/extras
  #   readOnly: true

extraContainers: []
  # - name: do-something
  #   image: busybox
  #   command: ['do', 'something']

extraInitContainers: []
  # - name: do-somethings
  #   image: busybox
  #   command: ['do', 'something']

# This is the PriorityClass settings as defined in
# https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass
priorityClassName: ""

# By default this will make sure two pods don't end up on the same node
# Changing this to a region would allow you to spread pods across regions
antiAffinityTopologyKey: "kubernetes.io/hostname"

# Hard means that by default pods will only be scheduled if there are enough nodes for them
# and that they will never end up on the same node. Setting this to soft will do this "best effort".
# Setting this to custom will use what is passed into customAntiAffinity.
antiAffinity: "soft"

# Allows passing in custom anti-affinity settings as defined in
# https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#types-of-inter-pod-affinity-and-anti-affinity
# Using this parameter requires setting antiAffinity to custom.
customAntiAffinity: {}

# This is the node affinity settings as defined in
# https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity-beta-feature
nodeAffinity: {}

# This is the pod affinity settings as defined in
# https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#types-of-inter-pod-affinity-and-anti-affinity
podAffinity: {}

# This is the pod topology spread constraints
# https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/
topologySpreadConstraints: []

# The default is to deploy all pods serially. By setting this to parallel all pods are started at
# the same time when bootstrapping the cluster
podManagementPolicy: "Parallel"

# The environment variables injected by service links are not used, but can lead to slow OpenSearch boot times when
# there are many services in the current namespace.
# If you experience slow pod startups you probably want to set this to `false`.
enableServiceLinks: true

protocol: https
httpPort: 9200
transportPort: 9300
metricsPort: 9600
httpHostPort: ""
transportHostPort: ""


service:
  labels: {}
  labelsHeadless: {}
  headless:
    annotations: {}
  type: ClusterIP
  # The IP family and IP families options are to set the behaviour in a dual-stack environment
  # Omitting these values will let the service fall back to whatever the CNI dictates the defaults
  # should be
  #
  # ipFamilyPolicy: SingleStack
  # ipFamilies:
  # - IPv4
  nodePort: ""
  annotations: {}
  httpPortName: http
  transportPortName: transport
  metricsPortName: metrics
  loadBalancerIP: ""
  loadBalancerSourceRanges: []
  externalTrafficPolicy: ""

updateStrategy: RollingUpdate

# This is the max unavailable setting for the pod disruption budget
# The default value of 1 will make sure that kubernetes won't allow more than 1
# of your pods to be unavailable during maintenance
maxUnavailable: 1

podSecurityContext:
  fsGroup: 1000
  runAsUser: 1000

securityContext:
  capabilities:
    drop:
      - ALL
  # readOnlyRootFilesystem: true
  runAsNonRoot: true
  runAsUser: 1000

securityConfig:
  enabled: true
  path: "/usr/share/opensearch/config/opensearch-security"
  actionGroupsSecret:
  configSecret:
  internalUsersSecret:
  rolesSecret:
  rolesMappingSecret:
  tenantsSecret:
  # The following option simplifies securityConfig by using a single secret and
  # specifying the config files as keys in the secret instead of creating
  # different secrets for for each config file.
  # Note that this is an alternative to the individual secret configuration
  # above and shouldn't be used if the above secrets are used.
  config:
    # There are multiple ways to define the configuration here:
    # * If you define anything under data, the chart will automatically create
    #   a secret and mount it. This is best option to choose if you want to override all the
    #   existing yml files at once.
    # * If you define securityConfigSecret, the chart will assume this secret is
    #   created externally and mount it. This is best option to choose if your intention is to
    #   only update a single yml file.
    # * It is an error to define both data and securityConfigSecret.
    securityConfigSecret: ""
    dataComplete: true
    data: {}
      # config.yml: |-
      # internal_users.yml: |-
      # roles.yml: |-
      # roles_mapping.yml: |-
      # action_groups.yml: |-
      # tenants.yml: |-

# How long to wait for opensearch to stop gracefully
terminationGracePeriod: 120

sysctlVmMaxMapCount: 262144

startupProbe:
  tcpSocket:
    port: 9200
  initialDelaySeconds: 5
  periodSeconds: 10
  timeoutSeconds: 3
  failureThreshold: 30

livenessProbe: {}
  # periodSeconds: 20
  # timeoutSeconds: 5
  # failureThreshold: 10
  # successThreshold: 1
  # initialDelaySeconds: 10
  # tcpSocket:
  #   port: 9200

readinessProbe:
  tcpSocket:
    port: 9200
  periodSeconds: 5
  timeoutSeconds: 3
  failureThreshold: 3

## Use an alternate scheduler.
## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/
##
schedulerName: ""

imagePullSecrets: []
nodeSelector: {}
tolerations: []

# Enabling this will publically expose your OpenSearch instance.
# Only enable this if you have security enabled on your cluster
ingress:
  enabled: false
  # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName
  # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress
  # ingressClassName: nginx

  annotations: {}
    # kubernetes.io/ingress.class: nginx
    # kubernetes.io/tls-acme: "true"
  ingressLabels: {}
  path: /
  hosts:
    - chart-example.local
  tls: []
  #  - secretName: chart-example-tls
  #    hosts:
  #      - chart-example.local

nameOverride: ""
fullnameOverride: ""

masterTerminationFix: false

opensearchLifecycle: {}
  # preStop:
  #   exec:
  #     command: ["/bin/sh", "-c", "echo Hello from the preStart handler > /usr/share/message"]
  # postStart:
  #   exec:
  #     command: ["/bin/sh", "-c", "echo Hello from the postStart handler > /usr/share/message"]

lifecycle: {}
  # preStop:
  #   exec:
  #     command: ["/bin/sh", "-c", "echo Hello from the postStart handler > /usr/share/message"]
  # postStart:
  #   exec:
  #     command:
  #       - bash
  #       - -c
  #       - |
  #         #!/bin/bash
  #         # Add a template to adjust number of shards/replicas1
  #         TEMPLATE_NAME=my_template
  #         INDEX_PATTERN="logstash-*"
  #         SHARD_COUNT=8
  #         REPLICA_COUNT=1
  #         ES_URL=http://localhost:9200
  #         while [[ "$(curl -s -o /dev/null -w '%{http_code}\n' $ES_URL)" != "200" ]]; do sleep 1; done
  #         curl -XPUT "$ES_URL/_template/$TEMPLATE_NAME" -H 'Content-Type: application/json' -d'{"index_patterns":['\""$INDEX_PATTERN"\"'],"settings":{"number_of_shards":'$SHARD_COUNT',"number_of_replicas":'$REPLICA_COUNT'}}'

keystore: []
# To add secrets to the keystore:
#  - secretName: opensearch-encryption-key

networkPolicy:
  create: false
  ## Enable creation of NetworkPolicy resources. Only Ingress traffic is filtered for now.
  ## In order for a Pod to access OpenSearch, it needs to have the following label:
  ## {{ template "uname" . }}-client: "true"
  ## Example for default configuration to access HTTP port:
  ## opensearch-master-http-client: "true"
  ## Example for default configuration to access transport port:
  ## opensearch-master-transport-client: "true"

  http:
    enabled: false

# Deprecated
# please use the above podSecurityContext.fsGroup instead
fsGroup: ""

## Set optimal sysctl's through securityContext. This requires privilege. Can be disabled if
## the system has already been preconfigured. (Ex: https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html)
## Also see: https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/
sysctl:
  enabled: false

## Set optimal sysctl's through privileged initContainer.
sysctlInit:
  enabled: false
  # override image, which is busybox by default
  # image: busybox
  # override image tag, which is latest by default
  # imageTag:

## Enable to add 3rd Party / Custom plugins not offered in the default OpenSearch image.
plugins:
  enabled: false
  installList: []
  # - example-fake-plugin

# -- Array of extra K8s manifests to deploy
extraObjects: []
  # - apiVersion: secrets-store.csi.x-k8s.io/v1
  #   kind: SecretProviderClass
  #   metadata:
  #     name: argocd-secrets-store
  #   spec:
  #     provider: aws
  #     parameters:
  #       objects: |
  #         - objectName: "argocd"
  #           objectType: "secretsmanager"
  #           jmesPath:
  #               - path: "client_id"
  #                 objectAlias: "client_id"
  #               - path: "client_secret"
  #                 objectAlias: "client_secret"
  #     secretObjects:
  #     - data:
  #       - key: client_id
  #         objectName: client_id
  #       - key: client_secret
  #         objectName: client_secret
  #       secretName: argocd-secrets-store
  #       type: Opaque
  #       labels:
  #         app.kubernetes.io/part-of: argocd
  # - |
  #    apiVersion: policy/v1
  #    kind: PodDisruptionBudget
  #    metadata:
  #      name: {{ template "opensearch.uname" . }}
  #      labels:
  #        {{- include "opensearch.labels" . | nindent 4 }}
  #    spec:
  #      minAvailable: 1
  #      selector:
  #        matchLabels:
  #          {{- include "opensearch.selectorLabels" . | nindent 6 }}

Relevant Logs or Screenshots:

Defaulted container "opensearch" out of: opensearch, fsgroup-volume (init), configfile (init)
Enabling OpenSearch Security Plugin
Enabling execution of install_demo_configuration.sh for OpenSearch Security Plugin 
OpenSearch 2.12.0 onwards, the OpenSearch Security Plugin a change that requires an initial password for 'admin' user. 
Please define an environment variable 'OPENSEARCH_INITIAL_ADMIN_PASSWORD' with a strong password string. 
If a password is not provided, the setup will quit. 
 For more details, please visit: https://opensearch.org/docs/latest/install-and-configure/install-opensearch/docker/
### OpenSearch Security Demo Installer
### ** Warning: Do not use on production or public reachable systems **
OpenSearch install type: rpm/deb on Linux 5.15.0-102-generic amd64
OpenSearch config dir: /usr/share/opensearch/config/
OpenSearch config file: /usr/share/opensearch/config/opensearch.yml
OpenSearch bin dir: /usr/share/opensearch/bin/
OpenSearch plugins dir: /usr/share/opensearch/plugins/
OpenSearch lib dir: /usr/share/opensearch/lib/
Detected OpenSearch Version: 2.13.0
Detected OpenSearch Security Version: 2.13.0.0
/usr/share/opensearch/config/opensearch.yml seems to be already configured for Security. Quit.
Enabling execution of OPENSEARCH_HOME/bin/opensearch-performance-analyzer/performance-analyzer-agent-cli for OpenSearch Performance Analyzer Plugin
WARNING: Using incubator modules: jdk.incubator.vector
WARNING: A terminally deprecated method in java.lang.System has been called
WARNING: System::setSecurityManager has been called by org.opensearch.bootstrap.OpenSearch (file:/usr/share/opensearch/lib/opensearch-2.13.0.jar)
WARNING: Please consider reporting this to the maintainers of org.opensearch.bootstrap.OpenSearch
WARNING: System::setSecurityManager will be removed in a future release
Apr 30, 2024 2:27:29 PM sun.util.locale.provider.LocaleProviderAdapter <clinit>
WARNING: COMPAT locale provider will be removed in a future release
WARNING: A terminally deprecated method in java.lang.System has been called
WARNING: System::setSecurityManager has been called by org.opensearch.bootstrap.Security (file:/usr/share/opensearch/lib/opensearch-2.13.0.jar)
WARNING: Please consider reporting this to the maintainers of org.opensearch.bootstrap.Security
WARNING: System::setSecurityManager will be removed in a future release
[2024-04-30T14:27:30,022][INFO ][o.o.n.Node               ] [opensearch-cluster-master-0] version[2.13.0], pid[35], build[tar/7ec678d1b7c87d6e779fdef94e33623e1f1e2647/2024-03-26T00:02:39.659767978Z], OS[Linux/5.15.0-102-generic/amd64], JVM[Eclipse Adoptium/OpenJDK 64-Bit Server VM/21.0.2/21.0.2+13-LTS]
[2024-04-30T14:27:30,024][INFO ][o.o.n.Node               ] [opensearch-cluster-master-0] JVM home [/usr/share/opensearch/jdk], using bundled JDK/JRE [true]
[2024-04-30T14:27:30,024][INFO ][o.o.n.Node               ] [opensearch-cluster-master-0] JVM arguments [-Xshare:auto, -Dopensearch.networkaddress.cache.ttl=60, -Dopensearch.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.security.manager=allow, -Djava.locale.providers=SPI,COMPAT, -Xms1g, -Xmx1g, -XX:+UseG1GC, -XX:G1ReservePercent=25, -XX:InitiatingHeapOccupancyPercent=30, -Djava.io.tmpdir=/tmp/opensearch-3094161089718262216, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Djava.security.manager=allow, --add-modules=jdk.incubator.vector, -Djava.util.concurrent.ForkJoinPool.common.threadFactory=org.opensearch.secure_sm.SecuredForkJoinWorkerThreadFactory, -Dclk.tck=100, -Djdk.attach.allowAttachSelf=true, -Djava.security.policy=/usr/share/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy, --add-opens=jdk.attach/sun.tools.attach=ALL-UNNAMED, -Dopensearch.cgroups.hierarchy.override=/, -Xmx512M, -Xms512M, -XX:MaxDirectMemorySize=268435456, -Dopensearch.path.home=/usr/share/opensearch, -Dopensearch.path.conf=/usr/share/opensearch/config, -Dopensearch.distribution.type=tar, -Dopensearch.bundled_jdk=true]
[2024-04-30T14:27:31,250][INFO ][o.o.s.s.t.SSLConfig      ] [opensearch-cluster-master-0] SSL dual mode is disabled
[2024-04-30T14:27:31,251][INFO ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-0] OpenSearch Config path is /usr/share/opensearch/config
[2024-04-30T14:27:31,479][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-0] JVM supports TLSv1.3
[2024-04-30T14:27:31,481][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-0] Config directory is /usr/share/opensearch/config/, from there the key- and truststore files are resolved relatively
[2024-04-30T14:27:31,494][ERROR][o.o.b.OpenSearchUncaughtExceptionHandler] [opensearch-cluster-master-0] uncaught exception in thread [main]
org.opensearch.bootstrap.StartupException: java.lang.IllegalStateException: failed to load plugin class [org.opensearch.security.OpenSearchSecurityPlugin]
        at org.opensearch.bootstrap.OpenSearch.init(OpenSearch.java:185) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.bootstrap.OpenSearch.execute(OpenSearch.java:172) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:104) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.cli.Command.mainWithoutErrorHandling(Command.java:138) ~[opensearch-cli-2.13.0.jar:2.13.0]
        at org.opensearch.cli.Command.main(Command.java:101) ~[opensearch-cli-2.13.0.jar:2.13.0]
        at org.opensearch.bootstrap.OpenSearch.main(OpenSearch.java:138) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.bootstrap.OpenSearch.main(OpenSearch.java:104) ~[opensearch-2.13.0.jar:2.13.0]
Caused by: java.lang.IllegalStateException: failed to load plugin class [org.opensearch.security.OpenSearchSecurityPlugin]
        at org.opensearch.plugins.PluginsService.loadPlugin(PluginsService.java:803) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.plugins.PluginsService.loadBundle(PluginsService.java:743) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.plugins.PluginsService.loadBundles(PluginsService.java:544) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.plugins.PluginsService.<init>(PluginsService.java:196) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.node.Node.<init>(Node.java:490) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.node.Node.<init>(Node.java:417) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:242) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.bootstrap.Bootstrap.setup(Bootstrap.java:242) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.bootstrap.Bootstrap.init(Bootstrap.java:404) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.bootstrap.OpenSearch.init(OpenSearch.java:181) ~[opensearch-2.13.0.jar:2.13.0]
        ... 6 more
uncaught exception in thread [main]
Caused by: java.lang.reflect.InvocationTargetException
        at java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:74) ~[?:?]
        at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:502) ~[?:?]
        at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:486) ~[?:?]
        at org.opensearch.plugins.PluginsService.loadPlugin(PluginsService.java:794) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.plugins.PluginsService.loadBundle(PluginsService.java:743) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.plugins.PluginsService.loadBundles(PluginsService.java:544) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.plugins.PluginsService.<init>(PluginsService.java:196) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.node.Node.<init>(Node.java:490) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.node.Node.<init>(Node.java:417) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:242) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.bootstrap.Bootstrap.setup(Bootstrap.java:242) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.bootstrap.Bootstrap.init(Bootstrap.java:404) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.bootstrap.OpenSearch.init(OpenSearch.java:181) ~[opensearch-2.13.0.jar:2.13.0]
        ... 6 more
Caused by: org.opensearch.OpenSearchSecurityException: Error while initializing transport SSL layer from PEM: OpenSearchException[Unable to read /usr/share/opensearch/config/esnode.pem (/usr/share/opensearch/config/esnode.pem). Please make sure this files exists and is readable regarding to permissions. Property: plugins.security.ssl.transport.pemcert_filepath]
        at org.opensearch.security.ssl.DefaultSecurityKeyStore.initTransportSSLConfig(DefaultSecurityKeyStore.java:484) ~[?:?]
        at org.opensearch.security.ssl.DefaultSecurityKeyStore.initSSLConfig(DefaultSecurityKeyStore.java:298) ~[?:?]
        at org.opensearch.security.ssl.DefaultSecurityKeyStore.<init>(DefaultSecurityKeyStore.java:204) ~[?:?]
        at org.opensearch.security.ssl.OpenSearchSecuritySSLPlugin.<init>(OpenSearchSecuritySSLPlugin.java:235) ~[?:?]
        at org.opensearch.security.OpenSearchSecurityPlugin.<init>(OpenSearchSecurityPlugin.java:295) ~[?:?]
        at java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:62) ~[?:?]
        at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:502) ~[?:?]
        at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:486) ~[?:?]
        at org.opensearch.plugins.PluginsService.loadPlugin(PluginsService.java:794) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.plugins.PluginsService.loadBundle(PluginsService.java:743) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.plugins.PluginsService.loadBundles(PluginsService.java:544) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.plugins.PluginsService.<init>(PluginsService.java:196) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.node.Node.<init>(Node.java:490) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.node.Node.<init>(Node.java:417) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:242) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.bootstrap.Bootstrap.setup(Bootstrap.java:242) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.bootstrap.Bootstrap.init(Bootstrap.java:404) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.bootstrap.OpenSearch.init(OpenSearch.java:181) ~[opensearch-2.13.0.jar:2.13.0]
        ... 6 more
Caused by: org.opensearch.OpenSearchException: Unable to read /usr/share/opensearch/config/esnode.pem (/usr/share/opensearch/config/esnode.pem). Please make sure this files exists and is readable regarding to permissions. Property: plugins.security.ssl.transport.pemcert_filepath
        at org.opensearch.security.ssl.DefaultSecurityKeyStore.checkPath(DefaultSecurityKeyStore.java:1135) ~[?:?]
        at org.opensearch.security.ssl.DefaultSecurityKeyStore.resolve(DefaultSecurityKeyStore.java:276) ~[?:?]
        at org.opensearch.security.ssl.DefaultSecurityKeyStore.initTransportSSLConfig(DefaultSecurityKeyStore.java:454) ~[?:?]
        at org.opensearch.security.ssl.DefaultSecurityKeyStore.initSSLConfig(DefaultSecurityKeyStore.java:298) ~[?:?]
        at org.opensearch.security.ssl.DefaultSecurityKeyStore.<init>(DefaultSecurityKeyStore.java:204) ~[?:?]
        at org.opensearch.security.ssl.OpenSearchSecuritySSLPlugin.<init>(OpenSearchSecuritySSLPlugin.java:235) ~[?:?]
        at org.opensearch.security.OpenSearchSecurityPlugin.<init>(OpenSearchSecurityPlugin.java:295) ~[?:?]
        at java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:62) ~[?:?]
        at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:502) ~[?:?]
        at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:486) ~[?:?]
        at org.opensearch.plugins.PluginsService.loadPlugin(PluginsService.java:794) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.plugins.PluginsService.loadBundle(PluginsService.java:743) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.plugins.PluginsService.loadBundles(PluginsService.java:544) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.plugins.PluginsService.<init>(PluginsService.java:196) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.node.Node.<init>(Node.java:490) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.node.Node.<init>(Node.java:417) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:242) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.bootstrap.Bootstrap.setup(Bootstrap.java:242) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.bootstrap.Bootstrap.init(Bootstrap.java:404) ~[opensearch-2.13.0.jar:2.13.0]
        at org.opensearch.bootstrap.OpenSearch.init(OpenSearch.java:181) ~[opensearch-2.13.0.jar:2.13.0]
        ... 6 more
java.lang.IllegalStateException: failed to load plugin class [org.opensearch.security.OpenSearchSecurityPlugin]
Likely root cause: OpenSearchException[Unable to read /usr/share/opensearch/config/esnode.pem (/usr/share/opensearch/config/esnode.pem). Please make sure this files exists and is readable regarding to permissions. Property: plugins.security.ssl.transport.pemcert_filepath]
        at org.opensearch.security.ssl.DefaultSecurityKeyStore.checkPath(DefaultSecurityKeyStore.java:1135)
        at org.opensearch.security.ssl.DefaultSecurityKeyStore.resolve(DefaultSecurityKeyStore.java:276)
        at org.opensearch.security.ssl.DefaultSecurityKeyStore.initTransportSSLConfig(DefaultSecurityKeyStore.java:454)
        at org.opensearch.security.ssl.DefaultSecurityKeyStore.initSSLConfig(DefaultSecurityKeyStore.java:298)
        at org.opensearch.security.ssl.DefaultSecurityKeyStore.<init>(DefaultSecurityKeyStore.java:204)
        at org.opensearch.security.ssl.OpenSearchSecuritySSLPlugin.<init>(OpenSearchSecuritySSLPlugin.java:235)
        at org.opensearch.security.OpenSearchSecurityPlugin.<init>(OpenSearchSecurityPlugin.java:295)
        at java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:62)
        at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:502)
        at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:486)
        at org.opensearch.plugins.PluginsService.loadPlugin(PluginsService.java:794)
        at org.opensearch.plugins.PluginsService.loadBundle(PluginsService.java:743)
        at org.opensearch.plugins.PluginsService.loadBundles(PluginsService.java:544)
        at org.opensearch.plugins.PluginsService.<init>(PluginsService.java:196)
        at org.opensearch.node.Node.<init>(Node.java:490)
        at org.opensearch.node.Node.<init>(Node.java:417)
        at org.opensearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:242)
        at org.opensearch.bootstrap.Bootstrap.setup(Bootstrap.java:242)
        at org.opensearch.bootstrap.Bootstrap.init(Bootstrap.java:404)
        at org.opensearch.bootstrap.OpenSearch.init(OpenSearch.java:181)
        at org.opensearch.bootstrap.OpenSearch.execute(OpenSearch.java:172)
        at org.opensearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:104)
        at org.opensearch.cli.Command.mainWithoutErrorHandling(Command.java:138)
        at org.opensearch.cli.Command.main(Command.java:101)
        at org.opensearch.bootstrap.OpenSearch.main(OpenSearch.java:138)
        at org.opensearch.bootstrap.OpenSearch.main(OpenSearch.java:104)
For complete error details, refer to the log at /usr/share/opensearch/logs/opensearch-cluster.log

Hi @abhyankar,

The first thing to address is your OPENSEARCH_INITIAL_ADMIN_PASSWORD is too weak, I believe it is too short cannot find the exact parameters but it has to be a strong password you can find more info here: security/DEVELOPER_GUIDE.md at main · opensearch-project/security · GitHub

Moreover, you will need to specify the following (in your values.yml):

    data: {}
      # config.yml: |-
      # internal_users.yml: |-
      # roles.yml: |-
      # roles_mapping.yml: |-
      # action_groups.yml: |-
      # tenants.yml: |-

Please see some guidance (demo) for the .yml files here: Modifying the YAML files - OpenSearch Documentation

Here is some sample data that I’ve used in the past if that helps:

data: 
  config.yml: |-
    _meta:
      type: "config"
      config_version: 2
    config:
      dynamic:
        http:
          anonymous_auth_enabled: false
          xff:
            enabled: false
            internalProxies: '192\.168\.0\.10|192\.168\.0\.11' # regex pattern
            remoteIpHeader:  'x-forwarded-for'
        authc:
          basic_internal_auth_domain:
            description: "Authenticate via HTTP Basic against internal users database"
            http_enabled: true
            transport_enabled: true
            order: 1 # Must set to 1 to prevent logs flooding with warnings
            http_authenticator:
              type: basic
              challenge: false
            authentication_backend:
              type: internal
          openid_auth_domain:
            http_enabled: true
            transport_enabled: true
            order: 0 # Must set to 0 to precede basic authentication or logs will flood with warnings
            http_authenticator:
              type: openid
              challenge: false
              config:
                subject_key: email # Users will be able to login with email and show it as username
                roles_key: roles # Use roles from App registrations, do not set to groups.
                openid_connect_url: https://login.microsoftonline.com/tenantid/v2.0/.well-known/openid-configuration # App registration IDP metadata file URL
                skip_users:
                  - admin
                  - kibanaserver
                  - kibanaro
                  - logstash
                  - readall
                  - snapshotrestore
            authentication_backend:
              type: noop # Here we disable authentication backend since we will be using Azure AD for that
  internal_users.yml: |-
        ---
        # This is the internal user database
        # The hash value is a bcrypt hash and can be generated with plugin/tools/hash.sh

        _meta:
          type: "internalusers"
          config_version: 2

        # Define your internal users here

        ## Demo users

        admin:
          hash: "$2a$12$VcCDgh2NDk07JGN0rjGbM.Ad41qVR/YFJcgHp0UGns5JDymv..TOG"
          reserved: true
          backend_roles:
          - "admin"
          description: "Demo admin user"

        anomalyadmin:
          hash: "$2y$12$TRwAAJgnNo67w3rVUz4FIeLx9Dy/llB79zf9I15CKJ9vkM4ZzAd3."
          reserved: false
          opendistro_security_roles:
          - "anomaly_full_access"
          description: "Demo anomaly admin user, using internal role"

        kibanaserver:
          hash: "$2a$12$4AcgAt3xwOWadA5s5blL6ev39OXDNhmOesEoo33eZtrq2N0YrU3H."
          reserved: true
          description: "Demo OpenSearch Dashboards user"

        kibanaro:
          hash: "$2a$12$JJSXNfTowz7Uu5ttXfeYpeYE0arACvcwlPBStB1F.MI7f0U9Z4DGC"
          reserved: false
          backend_roles:
          - "kibanauser"
          - "readall"
          attributes:
            attribute1: "value1"
            attribute2: "value2"
            attribute3: "value3"
          description: "Demo OpenSearch Dashboards read only user, using external role mapping"

        logstash:
          hash: "$2a$12$u1ShR4l4uBS3Uv59Pa2y5.1uQuZBrZtmNfqB3iM/.jL0XoV9sghS2"
          reserved: false
          backend_roles:
          - "logstash"
          description: "Demo logstash user, using external role mapping"

        readall:
          hash: "$2a$12$ae4ycwzwvLtZxwZ82RmiEunBbIPiAmGZduBAjKN0TXdwQFtCwARz2"
          reserved: false
          backend_roles:
          - "readall"
          description: "Demo readall user, using external role mapping"

        snapshotrestore:
          hash: "$2y$12$DpwmetHKwgYnorbgdvORCenv4NAK8cPUg8AI6pxLCuWf/ALc0.v7W"
          reserved: false
          backend_roles:
          - "snapshotrestore"
          description: "Demo snapshotrestore user, using external role mapping"
    
  roles.yml: |-
        ---
        _meta:
          type: "roles"
          config_version: 2
        alerting_ack_alerts:
          reserved: true
          cluster_permissions:
          - "cluster:admin/opendistro/alerting/alerts/*"
        alerting_full_access:
          reserved: true
          cluster_permissions:
          - "cluster_monitor"
          - "cluster:admin/opendistro/alerting/*"
          index_permissions:
          - index_patterns:
            - "*"
            allowed_actions:
            - "indices_monitor"
            - "indices:admin/aliases/get"
            - "indices:admin/mappings/get"
        anomaly_read_access:
          reserved: true
          cluster_permissions:
          - "cluster:admin/opendistro/ad/detector/info"
          - "cluster:admin/opendistro/ad/detector/search"
          - "cluster:admin/opendistro/ad/detectors/get"
          - "cluster:admin/opendistro/ad/result/search"
        anomaly_full_access:
          reserved: true
          cluster_permissions:
          - "cluster_monitor"
          - "cluster:admin/opendistro/ad/*"
          index_permissions:
          - index_patterns:
            - "*"
            allowed_actions:
            - "indices_monitor"
            - "indices:admin/aliases/get"
            - "indices:admin/mappings/get"
        reports_instances_read_access:
          reserved: true
          cluster_permissions:
          - "cluster:admin/opendistro/reports/instance/list"
          - "cluster:admin/opendistro/reports/instance/get"
          - "cluster:admin/opendistro/reports/menu/download"
        reports_read_access:
          reserved: true
          cluster_permissions:
          - "cluster:admin/opendistro/reports/definition/get"
          - "cluster:admin/opendistro/reports/definition/list"
          - "cluster:admin/opendistro/reports/instance/list"
          - "cluster:admin/opendistro/reports/instance/get"
          - "cluster:admin/opendistro/reports/menu/download"
        reports_full_access:
          reserved: true
          cluster_permissions:
          - "cluster:admin/opendistro/reports/definition/create"
          - "cluster:admin/opendistro/reports/definition/update"
          - "cluster:admin/opendistro/reports/definition/on_demand"
          - "cluster:admin/opendistro/reports/definition/delete"
          - "cluster:admin/opendistro/reports/definition/get"
          - "cluster:admin/opendistro/reports/definition/list"
          - "cluster:admin/opendistro/reports/instance/list"
          - "cluster:admin/opendistro/reports/instance/get"
          - "cluster:admin/opendistro/reports/menu/download"
        SYSTEME_ABC-XYZ:
          reserved: false
          hidden: false
          cluster_permissions:
          - 'cluster_composite_ops'
          index_permissions:
          - index_patterns:
            - 'abc*logs-socles*'
            fls: []
            dls: ""
            masked_fields: []
            allowed_actions:
            - 'read'
            - 'kibana_all_read'
          - index_patterns:
            - 'abc*logs-2*'
            fls: []
            dls: ""
            masked_fields: []
            allowed_actions:
            - 'read'
            - 'kibana_all_read'
          - index_patterns:
            - '.kibana'
            fls: []
            dls: ""
            masked_fields: []
            allowed_actions:
            - 'read'
            - 'kibana_all_read'
    
  roles_mapping.yml: |-
        ---
        # In this file users, backendroles and hosts can be mapped to Security roles.
        # Permissions for OpenSearch roles are configured in roles.yml

        _meta:
          type: "rolesmapping"
          config_version: 2

        # Define your roles mapping here

        ## Demo roles mapping

        all_access:
          reserved: false
          backend_roles:
          - "admin"
          description: "Maps admin to all_access"

        own_index:
          reserved: false
          users:
          - "*"
          description: "Allow full access to an index named like the username"

        logstash:
          reserved: false
          backend_roles:
          - "logstash"

        kibana_user:
          reserved: false
          backend_roles:
          - "kibanauser"
          description: "Maps kibanauser to kibana_user"

        readall:
          reserved: false
          backend_roles:
          - "readall"

        manage_snapshots:
          reserved: false
          backend_roles:
          - "snapshotrestore"

        kibana_server:
          reserved: true
          users:
          - "kibanaserver"
    
  action_groups.yml: |-
    _meta:
      type: "actiongroups"
      config_version: 2

  tenants.yml: |-
        ---
        _meta:
          type: "tenants"
          config_version: 2

        # Define your tenants here

        ## Demo tenants
        admin_tenant:
          reserved: false
          description: "Demo tenant for admin user"

Best,
mj