Versions (relevant - OpenSearch/Dashboard/Server OS/Browser):
helm repo add opensearch helm-charts | ☸ A community repository for Helm Charts of OpenSearch Project.
helm repo update
helm search repo opensearch
helm install my-deployment opensearch/opensearch --values values.yaml
Describe the issue:
After deploying in k8s using helm, getting security related exceptions.
opensearch-cluster-master-0 status is orange
opensearch-cluster-master-1 status is green
opensearch-cluster-master-2 status is orange
Configuration:
values.yaml =
---
clusterName: "opensearch-cluster"
nodeGroup: "master"
# If discovery.type in the opensearch configuration is set to "single-node",
# this should be set to "true"
# If "true", replicas will be forced to 1
singleNode: false
# The service that non master groups will try to connect to when joining the cluster
# This should be set to clusterName + "-" + nodeGroup for your master group
masterService: "opensearch-cluster-master"
# OpenSearch roles that will be applied to this nodeGroup
# These will be set as environment variable "node.roles". E.g. node.roles=master,ingest,data,remote_cluster_client
roles:
- master
- ingest
- data
- remote_cluster_client
replicas: 3
# if not set, falls back to parsing .Values.imageTag, then .Chart.appVersion.
majorVersion: ""
global:
# Set if you want to change the default docker registry, e.g. a private one.
dockerRegistry: ""
# Allows you to add any config files in {{ .Values.opensearchHome }}/config
opensearchHome: /usr/share/opensearch
# such as opensearch.yml and log4j2.properties
config:
# Values must be YAML literal style scalar / YAML multiline string.
# <filename>: |
# <formatted-value(s)>
# log4j2.properties: |
# status = error
#
# appender.console.type = Console
# appender.console.name = console
# appender.console.layout.type = PatternLayout
# appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n
#
# rootLogger.level = info
# rootLogger.appenderRef.console.ref = console
opensearch.yml: |
cluster.name: opensearch-cluster
# Bind to all interfaces because we don't know what IP address Docker will assign to us.
network.host: 0.0.0.0
# Setting network.host to a non-loopback address enables the annoying bootstrap checks. "Single-node" mode disables them again.
# Implicitly done if ".singleNode" is set to "true".
# discovery.type: single-node
# Start OpenSearch Security Demo Configuration
# WARNING: revise all the lines below before you go into production
plugins:
security:
ssl:
transport:
pemcert_filepath: esnode.pem
pemkey_filepath: esnode-key.pem
pemtrustedcas_filepath: root-ca.pem
enforce_hostname_verification: false
http:
enabled: true
pemcert_filepath: esnode.pem
pemkey_filepath: esnode-key.pem
pemtrustedcas_filepath: root-ca.pem
allow_unsafe_democertificates: true
allow_default_init_securityindex: true
authcz:
admin_dn:
- CN=kirk,OU=client,O=client,L=test,C=de
audit.type: internal_opensearch
enable_snapshot_restore_privilege: true
check_snapshot_restore_write_privileges: true
restapi:
roles_enabled: ["all_access", "security_rest_api_access"]
system_indices:
enabled: true
indices:
[
".opendistro-alerting-config",
".opendistro-alerting-alert*",
".opendistro-anomaly-results*",
".opendistro-anomaly-detector*",
".opendistro-anomaly-checkpoints",
".opendistro-anomaly-detection-state",
".opendistro-reports-*",
".opendistro-notifications-*",
".opendistro-notebooks",
".opendistro-asynchronous-search-response*",
]
######## End OpenSearch Security Demo Configuration ########
# log4j2.properties:
# Extra environment variables to append to this nodeGroup
# This will be appended to the current 'env:' key. You can use any of the kubernetes env
# syntax here
extraEnvs:
# - name: MY_ENVIRONMENT_VAR
# value: the_value_goes_here
# Chart version 2.18.0 and App Version OpenSearch 2.12.0 onwards a custom strong password needs to be provided in order to setup demo admin user.
# Cluster will not spin-up without this unless demo config install is disabled.
- name: OPENSEARCH_INITIAL_ADMIN_PASSWORD
value: "aHo1@pizZ#"
# Allows you to load environment variables from kubernetes secret or config map
envFrom: []
# - secretRef:
# name: env-secret
# - configMapRef:
# name: config-map
# A list of secrets and their paths to mount inside the pod
# This is useful for mounting certificates for security and for mounting
# the X-Pack license
secretMounts: []
hostAliases: []
# - ip: "127.0.0.1"
# hostnames:
# - "foo.local"
# - "bar.local"
image:
repository: "opensearchproject/opensearch"
# override image tag, which is .Chart.AppVersion by default
tag: ""
pullPolicy: "IfNotPresent"
podAnnotations: {}
# iam.amazonaws.com/role: es-cluster
# OpenSearch Statefulset annotations
openSearchAnnotations: {}
# additionals labels
labels: {}
opensearchJavaOpts: "-Xmx512M -Xms512M"
resources:
requests:
cpu: "1000m"
memory: "100Mi"
initResources: {}
# limits:
# cpu: "25m"
# memory: "128Mi"
# requests:
# cpu: "25m"
# memory: "128Mi"
sidecarResources: {}
# limits:
# cpu: "25m"
# memory: "128Mi"
# requests:
# cpu: "25m"
# memory: "128Mi"
networkHost: "0.0.0.0"
rbac:
create: false
serviceAccountAnnotations: {}
serviceAccountName: ""
# Controls whether or not the Service Account token is automatically mounted to /var/run/secrets/kubernetes.io/serviceaccount
automountServiceAccountToken: false
podSecurityPolicy:
create: false
name: ""
spec:
privileged: true
fsGroup:
rule: RunAsAny
runAsUser:
rule: RunAsAny
seLinux:
rule: RunAsAny
supplementalGroups:
rule: RunAsAny
volumes:
- secret
- configMap
- persistentVolumeClaim
- emptyDir
persistence:
enabled: true
# Set to false to disable the `fsgroup-volume` initContainer that will update permissions on the persistent disk.
enableInitChown: true
# override image, which is busybox by default
# image: busybox
# override image tag, which is latest by default
# imageTag:
labels:
# Add default labels for the volumeClaimTemplate of the StatefulSet
enabled: false
# OpenSearch Persistent Volume Storage Class
# If defined, storageClassName: <storageClass>
# If set to "-", storageClassName: "", which disables dynamic provisioning
# If undefined (the default) or set to null, no storageClassName spec is
# set, choosing the default provisioner. (gp2 on AWS, standard on
# GKE, AWS & OpenStack)
#
# storageClass: "-"
accessModes:
- ReadWriteOnce
size: 8Gi
annotations: {}
extraVolumes: []
# - name: extras
# emptyDir: {}
extraVolumeMounts: []
# - name: extras
# mountPath: /usr/share/extras
# readOnly: true
extraContainers: []
# - name: do-something
# image: busybox
# command: ['do', 'something']
extraInitContainers: []
# - name: do-somethings
# image: busybox
# command: ['do', 'something']
# This is the PriorityClass settings as defined in
# https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass
priorityClassName: ""
# By default this will make sure two pods don't end up on the same node
# Changing this to a region would allow you to spread pods across regions
antiAffinityTopologyKey: "kubernetes.io/hostname"
# Hard means that by default pods will only be scheduled if there are enough nodes for them
# and that they will never end up on the same node. Setting this to soft will do this "best effort".
# Setting this to custom will use what is passed into customAntiAffinity.
antiAffinity: "soft"
# Allows passing in custom anti-affinity settings as defined in
# https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#types-of-inter-pod-affinity-and-anti-affinity
# Using this parameter requires setting antiAffinity to custom.
customAntiAffinity: {}
# This is the node affinity settings as defined in
# https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity-beta-feature
nodeAffinity: {}
# This is the pod affinity settings as defined in
# https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#types-of-inter-pod-affinity-and-anti-affinity
podAffinity: {}
# This is the pod topology spread constraints
# https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/
topologySpreadConstraints: []
# The default is to deploy all pods serially. By setting this to parallel all pods are started at
# the same time when bootstrapping the cluster
podManagementPolicy: "Parallel"
# The environment variables injected by service links are not used, but can lead to slow OpenSearch boot times when
# there are many services in the current namespace.
# If you experience slow pod startups you probably want to set this to `false`.
enableServiceLinks: true
protocol: https
httpPort: 9200
transportPort: 9300
metricsPort: 9600
httpHostPort: ""
transportHostPort: ""
service:
labels: {}
labelsHeadless: {}
headless:
annotations: {}
type: ClusterIP
# The IP family and IP families options are to set the behaviour in a dual-stack environment
# Omitting these values will let the service fall back to whatever the CNI dictates the defaults
# should be
#
# ipFamilyPolicy: SingleStack
# ipFamilies:
# - IPv4
nodePort: ""
annotations: {}
httpPortName: http
transportPortName: transport
metricsPortName: metrics
loadBalancerIP: ""
loadBalancerSourceRanges: []
externalTrafficPolicy: ""
updateStrategy: RollingUpdate
# This is the max unavailable setting for the pod disruption budget
# The default value of 1 will make sure that kubernetes won't allow more than 1
# of your pods to be unavailable during maintenance
maxUnavailable: 1
podSecurityContext:
fsGroup: 1000
runAsUser: 1000
securityContext:
capabilities:
drop:
- ALL
# readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 1000
securityConfig:
enabled: true
path: "/usr/share/opensearch/config/opensearch-security"
actionGroupsSecret:
configSecret:
internalUsersSecret:
rolesSecret:
rolesMappingSecret:
tenantsSecret:
# The following option simplifies securityConfig by using a single secret and
# specifying the config files as keys in the secret instead of creating
# different secrets for for each config file.
# Note that this is an alternative to the individual secret configuration
# above and shouldn't be used if the above secrets are used.
config:
# There are multiple ways to define the configuration here:
# * If you define anything under data, the chart will automatically create
# a secret and mount it. This is best option to choose if you want to override all the
# existing yml files at once.
# * If you define securityConfigSecret, the chart will assume this secret is
# created externally and mount it. This is best option to choose if your intention is to
# only update a single yml file.
# * It is an error to define both data and securityConfigSecret.
securityConfigSecret: ""
dataComplete: true
data: {}
# config.yml: |-
# internal_users.yml: |-
# roles.yml: |-
# roles_mapping.yml: |-
# action_groups.yml: |-
# tenants.yml: |-
# How long to wait for opensearch to stop gracefully
terminationGracePeriod: 120
sysctlVmMaxMapCount: 262144
startupProbe:
tcpSocket:
port: 9200
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 3
failureThreshold: 30
livenessProbe: {}
# periodSeconds: 20
# timeoutSeconds: 5
# failureThreshold: 10
# successThreshold: 1
# initialDelaySeconds: 10
# tcpSocket:
# port: 9200
readinessProbe:
tcpSocket:
port: 9200
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 3
## Use an alternate scheduler.
## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/
##
schedulerName: ""
imagePullSecrets: []
nodeSelector: {}
tolerations: []
# Enabling this will publically expose your OpenSearch instance.
# Only enable this if you have security enabled on your cluster
ingress:
enabled: false
# For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName
# See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress
# ingressClassName: nginx
annotations: {}
# kubernetes.io/ingress.class: nginx
# kubernetes.io/tls-acme: "true"
ingressLabels: {}
path: /
hosts:
- chart-example.local
tls: []
# - secretName: chart-example-tls
# hosts:
# - chart-example.local
nameOverride: ""
fullnameOverride: ""
masterTerminationFix: false
opensearchLifecycle: {}
# preStop:
# exec:
# command: ["/bin/sh", "-c", "echo Hello from the preStart handler > /usr/share/message"]
# postStart:
# exec:
# command: ["/bin/sh", "-c", "echo Hello from the postStart handler > /usr/share/message"]
lifecycle: {}
# preStop:
# exec:
# command: ["/bin/sh", "-c", "echo Hello from the postStart handler > /usr/share/message"]
# postStart:
# exec:
# command:
# - bash
# - -c
# - |
# #!/bin/bash
# # Add a template to adjust number of shards/replicas1
# TEMPLATE_NAME=my_template
# INDEX_PATTERN="logstash-*"
# SHARD_COUNT=8
# REPLICA_COUNT=1
# ES_URL=http://localhost:9200
# while [[ "$(curl -s -o /dev/null -w '%{http_code}\n' $ES_URL)" != "200" ]]; do sleep 1; done
# curl -XPUT "$ES_URL/_template/$TEMPLATE_NAME" -H 'Content-Type: application/json' -d'{"index_patterns":['\""$INDEX_PATTERN"\"'],"settings":{"number_of_shards":'$SHARD_COUNT',"number_of_replicas":'$REPLICA_COUNT'}}'
keystore: []
# To add secrets to the keystore:
# - secretName: opensearch-encryption-key
networkPolicy:
create: false
## Enable creation of NetworkPolicy resources. Only Ingress traffic is filtered for now.
## In order for a Pod to access OpenSearch, it needs to have the following label:
## {{ template "uname" . }}-client: "true"
## Example for default configuration to access HTTP port:
## opensearch-master-http-client: "true"
## Example for default configuration to access transport port:
## opensearch-master-transport-client: "true"
http:
enabled: false
# Deprecated
# please use the above podSecurityContext.fsGroup instead
fsGroup: ""
## Set optimal sysctl's through securityContext. This requires privilege. Can be disabled if
## the system has already been preconfigured. (Ex: https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html)
## Also see: https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/
sysctl:
enabled: false
## Set optimal sysctl's through privileged initContainer.
sysctlInit:
enabled: false
# override image, which is busybox by default
# image: busybox
# override image tag, which is latest by default
# imageTag:
## Enable to add 3rd Party / Custom plugins not offered in the default OpenSearch image.
plugins:
enabled: false
installList: []
# - example-fake-plugin
# -- Array of extra K8s manifests to deploy
extraObjects: []
# - apiVersion: secrets-store.csi.x-k8s.io/v1
# kind: SecretProviderClass
# metadata:
# name: argocd-secrets-store
# spec:
# provider: aws
# parameters:
# objects: |
# - objectName: "argocd"
# objectType: "secretsmanager"
# jmesPath:
# - path: "client_id"
# objectAlias: "client_id"
# - path: "client_secret"
# objectAlias: "client_secret"
# secretObjects:
# - data:
# - key: client_id
# objectName: client_id
# - key: client_secret
# objectName: client_secret
# secretName: argocd-secrets-store
# type: Opaque
# labels:
# app.kubernetes.io/part-of: argocd
# - |
# apiVersion: policy/v1
# kind: PodDisruptionBudget
# metadata:
# name: {{ template "opensearch.uname" . }}
# labels:
# {{- include "opensearch.labels" . | nindent 4 }}
# spec:
# minAvailable: 1
# selector:
# matchLabels:
# {{- include "opensearch.selectorLabels" . | nindent 6 }}
Relevant Logs or Screenshots:
Defaulted container "opensearch" out of: opensearch, fsgroup-volume (init), configfile (init)
Enabling OpenSearch Security Plugin
Enabling execution of install_demo_configuration.sh for OpenSearch Security Plugin
OpenSearch 2.12.0 onwards, the OpenSearch Security Plugin a change that requires an initial password for 'admin' user.
Please define an environment variable 'OPENSEARCH_INITIAL_ADMIN_PASSWORD' with a strong password string.
If a password is not provided, the setup will quit.
For more details, please visit: https://opensearch.org/docs/latest/install-and-configure/install-opensearch/docker/
### OpenSearch Security Demo Installer
### ** Warning: Do not use on production or public reachable systems **
OpenSearch install type: rpm/deb on Linux 5.15.0-102-generic amd64
OpenSearch config dir: /usr/share/opensearch/config/
OpenSearch config file: /usr/share/opensearch/config/opensearch.yml
OpenSearch bin dir: /usr/share/opensearch/bin/
OpenSearch plugins dir: /usr/share/opensearch/plugins/
OpenSearch lib dir: /usr/share/opensearch/lib/
Detected OpenSearch Version: 2.13.0
Detected OpenSearch Security Version: 2.13.0.0
/usr/share/opensearch/config/opensearch.yml seems to be already configured for Security. Quit.
Enabling execution of OPENSEARCH_HOME/bin/opensearch-performance-analyzer/performance-analyzer-agent-cli for OpenSearch Performance Analyzer Plugin
WARNING: Using incubator modules: jdk.incubator.vector
WARNING: A terminally deprecated method in java.lang.System has been called
WARNING: System::setSecurityManager has been called by org.opensearch.bootstrap.OpenSearch (file:/usr/share/opensearch/lib/opensearch-2.13.0.jar)
WARNING: Please consider reporting this to the maintainers of org.opensearch.bootstrap.OpenSearch
WARNING: System::setSecurityManager will be removed in a future release
Apr 30, 2024 2:27:29 PM sun.util.locale.provider.LocaleProviderAdapter <clinit>
WARNING: COMPAT locale provider will be removed in a future release
WARNING: A terminally deprecated method in java.lang.System has been called
WARNING: System::setSecurityManager has been called by org.opensearch.bootstrap.Security (file:/usr/share/opensearch/lib/opensearch-2.13.0.jar)
WARNING: Please consider reporting this to the maintainers of org.opensearch.bootstrap.Security
WARNING: System::setSecurityManager will be removed in a future release
[2024-04-30T14:27:30,022][INFO ][o.o.n.Node ] [opensearch-cluster-master-0] version[2.13.0], pid[35], build[tar/7ec678d1b7c87d6e779fdef94e33623e1f1e2647/2024-03-26T00:02:39.659767978Z], OS[Linux/5.15.0-102-generic/amd64], JVM[Eclipse Adoptium/OpenJDK 64-Bit Server VM/21.0.2/21.0.2+13-LTS]
[2024-04-30T14:27:30,024][INFO ][o.o.n.Node ] [opensearch-cluster-master-0] JVM home [/usr/share/opensearch/jdk], using bundled JDK/JRE [true]
[2024-04-30T14:27:30,024][INFO ][o.o.n.Node ] [opensearch-cluster-master-0] JVM arguments [-Xshare:auto, -Dopensearch.networkaddress.cache.ttl=60, -Dopensearch.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.security.manager=allow, -Djava.locale.providers=SPI,COMPAT, -Xms1g, -Xmx1g, -XX:+UseG1GC, -XX:G1ReservePercent=25, -XX:InitiatingHeapOccupancyPercent=30, -Djava.io.tmpdir=/tmp/opensearch-3094161089718262216, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Djava.security.manager=allow, --add-modules=jdk.incubator.vector, -Djava.util.concurrent.ForkJoinPool.common.threadFactory=org.opensearch.secure_sm.SecuredForkJoinWorkerThreadFactory, -Dclk.tck=100, -Djdk.attach.allowAttachSelf=true, -Djava.security.policy=/usr/share/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy, --add-opens=jdk.attach/sun.tools.attach=ALL-UNNAMED, -Dopensearch.cgroups.hierarchy.override=/, -Xmx512M, -Xms512M, -XX:MaxDirectMemorySize=268435456, -Dopensearch.path.home=/usr/share/opensearch, -Dopensearch.path.conf=/usr/share/opensearch/config, -Dopensearch.distribution.type=tar, -Dopensearch.bundled_jdk=true]
[2024-04-30T14:27:31,250][INFO ][o.o.s.s.t.SSLConfig ] [opensearch-cluster-master-0] SSL dual mode is disabled
[2024-04-30T14:27:31,251][INFO ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-0] OpenSearch Config path is /usr/share/opensearch/config
[2024-04-30T14:27:31,479][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-0] JVM supports TLSv1.3
[2024-04-30T14:27:31,481][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-0] Config directory is /usr/share/opensearch/config/, from there the key- and truststore files are resolved relatively
[2024-04-30T14:27:31,494][ERROR][o.o.b.OpenSearchUncaughtExceptionHandler] [opensearch-cluster-master-0] uncaught exception in thread [main]
org.opensearch.bootstrap.StartupException: java.lang.IllegalStateException: failed to load plugin class [org.opensearch.security.OpenSearchSecurityPlugin]
at org.opensearch.bootstrap.OpenSearch.init(OpenSearch.java:185) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.bootstrap.OpenSearch.execute(OpenSearch.java:172) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:104) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.cli.Command.mainWithoutErrorHandling(Command.java:138) ~[opensearch-cli-2.13.0.jar:2.13.0]
at org.opensearch.cli.Command.main(Command.java:101) ~[opensearch-cli-2.13.0.jar:2.13.0]
at org.opensearch.bootstrap.OpenSearch.main(OpenSearch.java:138) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.bootstrap.OpenSearch.main(OpenSearch.java:104) ~[opensearch-2.13.0.jar:2.13.0]
Caused by: java.lang.IllegalStateException: failed to load plugin class [org.opensearch.security.OpenSearchSecurityPlugin]
at org.opensearch.plugins.PluginsService.loadPlugin(PluginsService.java:803) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.plugins.PluginsService.loadBundle(PluginsService.java:743) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.plugins.PluginsService.loadBundles(PluginsService.java:544) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.plugins.PluginsService.<init>(PluginsService.java:196) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.node.Node.<init>(Node.java:490) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.node.Node.<init>(Node.java:417) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:242) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.bootstrap.Bootstrap.setup(Bootstrap.java:242) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.bootstrap.Bootstrap.init(Bootstrap.java:404) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.bootstrap.OpenSearch.init(OpenSearch.java:181) ~[opensearch-2.13.0.jar:2.13.0]
... 6 more
uncaught exception in thread [main]
Caused by: java.lang.reflect.InvocationTargetException
at java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:74) ~[?:?]
at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:502) ~[?:?]
at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:486) ~[?:?]
at org.opensearch.plugins.PluginsService.loadPlugin(PluginsService.java:794) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.plugins.PluginsService.loadBundle(PluginsService.java:743) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.plugins.PluginsService.loadBundles(PluginsService.java:544) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.plugins.PluginsService.<init>(PluginsService.java:196) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.node.Node.<init>(Node.java:490) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.node.Node.<init>(Node.java:417) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:242) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.bootstrap.Bootstrap.setup(Bootstrap.java:242) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.bootstrap.Bootstrap.init(Bootstrap.java:404) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.bootstrap.OpenSearch.init(OpenSearch.java:181) ~[opensearch-2.13.0.jar:2.13.0]
... 6 more
Caused by: org.opensearch.OpenSearchSecurityException: Error while initializing transport SSL layer from PEM: OpenSearchException[Unable to read /usr/share/opensearch/config/esnode.pem (/usr/share/opensearch/config/esnode.pem). Please make sure this files exists and is readable regarding to permissions. Property: plugins.security.ssl.transport.pemcert_filepath]
at org.opensearch.security.ssl.DefaultSecurityKeyStore.initTransportSSLConfig(DefaultSecurityKeyStore.java:484) ~[?:?]
at org.opensearch.security.ssl.DefaultSecurityKeyStore.initSSLConfig(DefaultSecurityKeyStore.java:298) ~[?:?]
at org.opensearch.security.ssl.DefaultSecurityKeyStore.<init>(DefaultSecurityKeyStore.java:204) ~[?:?]
at org.opensearch.security.ssl.OpenSearchSecuritySSLPlugin.<init>(OpenSearchSecuritySSLPlugin.java:235) ~[?:?]
at org.opensearch.security.OpenSearchSecurityPlugin.<init>(OpenSearchSecurityPlugin.java:295) ~[?:?]
at java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:62) ~[?:?]
at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:502) ~[?:?]
at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:486) ~[?:?]
at org.opensearch.plugins.PluginsService.loadPlugin(PluginsService.java:794) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.plugins.PluginsService.loadBundle(PluginsService.java:743) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.plugins.PluginsService.loadBundles(PluginsService.java:544) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.plugins.PluginsService.<init>(PluginsService.java:196) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.node.Node.<init>(Node.java:490) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.node.Node.<init>(Node.java:417) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:242) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.bootstrap.Bootstrap.setup(Bootstrap.java:242) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.bootstrap.Bootstrap.init(Bootstrap.java:404) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.bootstrap.OpenSearch.init(OpenSearch.java:181) ~[opensearch-2.13.0.jar:2.13.0]
... 6 more
Caused by: org.opensearch.OpenSearchException: Unable to read /usr/share/opensearch/config/esnode.pem (/usr/share/opensearch/config/esnode.pem). Please make sure this files exists and is readable regarding to permissions. Property: plugins.security.ssl.transport.pemcert_filepath
at org.opensearch.security.ssl.DefaultSecurityKeyStore.checkPath(DefaultSecurityKeyStore.java:1135) ~[?:?]
at org.opensearch.security.ssl.DefaultSecurityKeyStore.resolve(DefaultSecurityKeyStore.java:276) ~[?:?]
at org.opensearch.security.ssl.DefaultSecurityKeyStore.initTransportSSLConfig(DefaultSecurityKeyStore.java:454) ~[?:?]
at org.opensearch.security.ssl.DefaultSecurityKeyStore.initSSLConfig(DefaultSecurityKeyStore.java:298) ~[?:?]
at org.opensearch.security.ssl.DefaultSecurityKeyStore.<init>(DefaultSecurityKeyStore.java:204) ~[?:?]
at org.opensearch.security.ssl.OpenSearchSecuritySSLPlugin.<init>(OpenSearchSecuritySSLPlugin.java:235) ~[?:?]
at org.opensearch.security.OpenSearchSecurityPlugin.<init>(OpenSearchSecurityPlugin.java:295) ~[?:?]
at java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:62) ~[?:?]
at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:502) ~[?:?]
at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:486) ~[?:?]
at org.opensearch.plugins.PluginsService.loadPlugin(PluginsService.java:794) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.plugins.PluginsService.loadBundle(PluginsService.java:743) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.plugins.PluginsService.loadBundles(PluginsService.java:544) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.plugins.PluginsService.<init>(PluginsService.java:196) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.node.Node.<init>(Node.java:490) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.node.Node.<init>(Node.java:417) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:242) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.bootstrap.Bootstrap.setup(Bootstrap.java:242) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.bootstrap.Bootstrap.init(Bootstrap.java:404) ~[opensearch-2.13.0.jar:2.13.0]
at org.opensearch.bootstrap.OpenSearch.init(OpenSearch.java:181) ~[opensearch-2.13.0.jar:2.13.0]
... 6 more
java.lang.IllegalStateException: failed to load plugin class [org.opensearch.security.OpenSearchSecurityPlugin]
Likely root cause: OpenSearchException[Unable to read /usr/share/opensearch/config/esnode.pem (/usr/share/opensearch/config/esnode.pem). Please make sure this files exists and is readable regarding to permissions. Property: plugins.security.ssl.transport.pemcert_filepath]
at org.opensearch.security.ssl.DefaultSecurityKeyStore.checkPath(DefaultSecurityKeyStore.java:1135)
at org.opensearch.security.ssl.DefaultSecurityKeyStore.resolve(DefaultSecurityKeyStore.java:276)
at org.opensearch.security.ssl.DefaultSecurityKeyStore.initTransportSSLConfig(DefaultSecurityKeyStore.java:454)
at org.opensearch.security.ssl.DefaultSecurityKeyStore.initSSLConfig(DefaultSecurityKeyStore.java:298)
at org.opensearch.security.ssl.DefaultSecurityKeyStore.<init>(DefaultSecurityKeyStore.java:204)
at org.opensearch.security.ssl.OpenSearchSecuritySSLPlugin.<init>(OpenSearchSecuritySSLPlugin.java:235)
at org.opensearch.security.OpenSearchSecurityPlugin.<init>(OpenSearchSecurityPlugin.java:295)
at java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:62)
at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:502)
at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:486)
at org.opensearch.plugins.PluginsService.loadPlugin(PluginsService.java:794)
at org.opensearch.plugins.PluginsService.loadBundle(PluginsService.java:743)
at org.opensearch.plugins.PluginsService.loadBundles(PluginsService.java:544)
at org.opensearch.plugins.PluginsService.<init>(PluginsService.java:196)
at org.opensearch.node.Node.<init>(Node.java:490)
at org.opensearch.node.Node.<init>(Node.java:417)
at org.opensearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:242)
at org.opensearch.bootstrap.Bootstrap.setup(Bootstrap.java:242)
at org.opensearch.bootstrap.Bootstrap.init(Bootstrap.java:404)
at org.opensearch.bootstrap.OpenSearch.init(OpenSearch.java:181)
at org.opensearch.bootstrap.OpenSearch.execute(OpenSearch.java:172)
at org.opensearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:104)
at org.opensearch.cli.Command.mainWithoutErrorHandling(Command.java:138)
at org.opensearch.cli.Command.main(Command.java:101)
at org.opensearch.bootstrap.OpenSearch.main(OpenSearch.java:138)
at org.opensearch.bootstrap.OpenSearch.main(OpenSearch.java:104)
For complete error details, refer to the log at /usr/share/opensearch/logs/opensearch-cluster.log