OpenSearch 2.9, OpenSearch Dashboards 2.9 running in docker
After a node restarts, Dashboards will give 401 after openid login.
Dashboards logs:
{"type":"log","@timestamp":"2023-09-13T12:52:41Z","tags":["error","plugins","securityDashboards"],"pid":1,"message":"OpenId authentication failed: Error: Authentication Exception"}
{"type":"response","@timestamp":"2023-09-13T12:52:40Z","tags":[],"pid":1,"method":"get","statusCode":401,"req":{"url":"/auth/openid/login?state=YFYM25e6siG4CkZixoZ0p3&session_state=2782af0b-5ea5-49f2-aeb4-2049d79e1fc5&code=58803566-f2fb-4432-b36e-66dea4fe0671.2782af0b-5ea5-49f2-aeb4-2049d79e1fc5.d021f693-b83c-45bf-aeaf-f7f624b96564","method":"get","headers":{"host":"dashboard:5669","user-agent":"Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/117.0","accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8","accept-language":"en-US,en;q=0.5","accept-encoding":"gzip, deflate, br","connection":"keep-alive","upgrade-insecure-requests":"1","sec-fetch-dest":"document","sec-fetch-mode":"navigate","sec-fetch-site":"none","sec-fetch-user":"?1"},"remoteAddress":"100.79.242.11","userAgent":"Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/117.0"},"res":{"statusCode":401,"responseTime":172,"contentLength":9},"message":"GET /auth/openid/login?state=YFYM25e6siG4CkZixoZ0p3&session_state=2782af0b-5ea5-49f2-aeb4-2049d79e1fc5&code=58803566-f2fb-4432-b36e-66dea4fe0671.2782af0b-5ea5-49f2-aeb4-2049d79e1fc5.d021f693-b83c-45bf-aeaf-f7f624b96564 401 172ms - 9.0B"}
{"type":"log","@timestamp":"2023-09-13T12:53:12Z","tags":["error","plugins","securityDashboards"],"pid":1,"message":"OpenId authentication failed: Error: Response Error: 400 Bad Request"}
{"type":"response","@timestamp":"2023-09-13T12:53:12Z","tags":[],"pid":1,"method":"get","statusCode":302,"req":{"url":"/auth/openid/login?state=YFYM25e6siG4CkZixoZ0p3&session_state=2782af0b-5ea5-49f2-aeb4-2049d79e1fc5&code=58803566-f2fb-4432-b36e-66dea4fe0671.2782af0b-5ea5-49f2-aeb4-2049d79e1fc5.d021f693-b83c-45bf-aeaf-f7f624b96564","method":"get","headers":{"host":"dashboard:5669","user-agent":"Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/117.0","accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8","accept-language":"en-US,en;q=0.5","accept-encoding":"gzip, deflate, br","connection":"keep-alive","upgrade-insecure-requests":"1","sec-fetch-dest":"document","sec-fetch-mode":"navigate","sec-fetch-site":"same-site"},"remoteAddress":"100.79.242.11","userAgent":"Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/117.0"},"res":{"statusCode":302,"responseTime":77,"contentLength":9},"message":"GET /auth/openid/login?state=YFYM25e6siG4CkZixoZ0p3&session_state=2782af0b-5ea5-49f2-aeb4-2049d79e1fc5&code=58803566-f2fb-4432-b36e-66dea4fe0671.2782af0b-5ea5-49f2-aeb4-2049d79e1fc5.d021f693-b83c-45bf-aeaf-f7f624b96564 302 77ms - 9.0B"}
{"type":"response","@timestamp":"2023-09-13T12:53:12Z","tags":[],"pid":1,"method":"get","statusCode":302,"req":{"url":"/auth/openid/login","method":"get","headers":{"host":"dashboard:5669","user-agent":"Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/117.0","accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8","accept-language":"en-US,en;q=0.5","accept-encoding":"gzip, deflate, br","connection":"keep-alive","upgrade-insecure-requests":"1","sec-fetch-dest":"document","sec-fetch-mode":"navigate","sec-fetch-site":"same-site"},"remoteAddress":"100.79.242.11","userAgent":"Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/117.0"},"res":{"statusCode":302,"responseTime":5,"contentLength":9},"message":"GET /auth/openid/login 302 5ms - 9.0B"}
{"type":"log","@timestamp":"2023-09-13T12:53:12Z","tags":["error","plugins","securityDashboards"],"pid":1,"message":"OpenId authentication failed: Error: Authentication Exception"}
{"type":"response","@timestamp":"2023-09-13T12:53:12Z","tags":[],"pid":1,"method":"get","statusCode":401,"req":{"url":"/auth/openid/login?state=JtVkx9Z0KGsm1D_BDZ1L3x&session_state=2782af0b-5ea5-49f2-aeb4-2049d79e1fc5&code=5bebb3e1-525b-4916-a3d4-28ed6c5cdff7.2782af0b-5ea5-49f2-aeb4-2049d79e1fc5.d021f693-b83c-45bf-aeaf-f7f624b96564","method":"get","headers":{"host":"dashboard:5669","user-agent":"Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/117.0","accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8","accept-language":"en-US,en;q=0.5","accept-encoding":"gzip, deflate, br","connection":"keep-alive","upgrade-insecure-requests":"1","sec-fetch-dest":"document","sec-fetch-mode":"navigate","sec-fetch-site":"same-site"},"remoteAddress":"100.79.242.11","userAgent":"Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/117.0"},"res":{"statusCode":401,"responseTime":55,"contentLength":9},"message":"GET /auth/openid/login?state=JtVkx9Z0KGsm1D_BDZ1L3x&session_state=2782af0b-5ea5-49f2-aeb4-2049d79e1fc5&code=5bebb3e1-525b-4916-a3d4-28ed6c5cdff7.2782af0b-5ea5-49f2-aeb4-2049d79e1fc5.d021f693-b83c-45bf-aeaf-f7f624b96564 401 55ms - 9.0B"}
In OpenSearch logs I see errors related to openid_connect_idp.pemkey_filepath and pemcert_filepath:
Caused by: java.lang.RuntimeException: com.amazon.dlic.util.SettingsBasedSSLConfigurator$SSLConfigException: Error loading PEM from /usr/share/opensearch/config/certificates/auth-client.key (openid_connect_idp.pemkey_filepath) for openid_connect_idp.
at com.amazon.dlic.auth.http.jwt.AbstractHTTPJwtAuthenticator.<init>(AbstractHTTPJwtAuthenticator.java:80) ~[opensearch-security-2.9.0.0.jar:2.9.0.0]
at com.amazon.dlic.auth.http.jwt.keybyoidc.HTTPJwtKeyByOpenIdConnectAuthenticator.<init>(HTTPJwtKeyByOpenIdConnectAuthenticator.java:26) ~[opensearch-security-2.9.0.0.jar:2.9.0.0]
at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
at org.opensearch.security.support.ReflectionHelper.instantiateAAA(ReflectionHelper.java:62) ~[opensearch-security-2.9.0.0.jar:2.9.0.0]
... 25 more
Caused by: com.amazon.dlic.util.SettingsBasedSSLConfigurator$SSLConfigException: Error loading PEM from /usr/share/opensearch/config/certificates/auth-client.key (openid_connect_idp.pemkey_filepath) for openid_connect_idp.
at com.amazon.dlic.util.SettingsBasedSSLConfigurator.initFromPem(SettingsBasedSSLConfigurator.java:296) ~[opensearch-security-2.9.0.0.jar:2.9.0.0]
at com.amazon.dlic.util.SettingsBasedSSLConfigurator.configureWithSettings(SettingsBasedSSLConfigurator.java:192) ~[opensearch-security-2.9.0.0.jar:2.9.0.0]
at com.amazon.dlic.util.SettingsBasedSSLConfigurator.buildSSLContext(SettingsBasedSSLConfigurator.java:115) ~[opensearch-security-2.9.0.0.jar:2.9.0.0]
at com.amazon.dlic.util.SettingsBasedSSLConfigurator.buildSSLConfig(SettingsBasedSSLConfigurator.java:129) ~[opensearch-security-2.9.0.0.jar:2.9.0.0]
at com.amazon.dlic.auth.http.jwt.keybyoidc.HTTPJwtKeyByOpenIdConnectAuthenticator.getSSLConfig(HTTPJwtKeyByOpenIdConnectAuthenticator.java:65) ~[opensearch-security-2.9.0.0.jar:2.9.0.0]
at com.amazon.dlic.auth.http.jwt.keybyoidc.HTTPJwtKeyByOpenIdConnectAuthenticator.initKeyProvider(HTTPJwtKeyByOpenIdConnectAuthenticator.java:47) ~[opensearch-security-2.9.0.0.jar:2.9.0.0]
at com.amazon.dlic.auth.http.jwt.AbstractHTTPJwtAuthenticator.<init>(AbstractHTTPJwtAuthenticator.java:75) ~[opensearch-security-2.9.0.0.jar:2.9.0.0]
at com.amazon.dlic.auth.http.jwt.keybyoidc.HTTPJwtKeyByOpenIdConnectAuthenticator.<init>(HTTPJwtKeyByOpenIdConnectAuthenticator.java:26) ~[opensearch-security-2.9.0.0.jar:2.9.0.0]
at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
at org.opensearch.security.support.ReflectionHelper.instantiateAAA(ReflectionHelper.java:62) ~[opensearch-security-2.9.0.0.jar:2.9.0.0]
... 25 more
Caused by: java.security.AccessControlException: access denied ("java.security.SecurityPermission" "removeProviderProperty.BC")
at java.security.AccessControlContext.checkPermission(AccessControlContext.java:485) ~[?:?]
at java.security.AccessController.checkPermission(AccessController.java:1068) ~[?:?]
at java.lang.SecurityManager.checkPermission(SecurityManager.java:416) ~[?:?]
at java.lang.SecurityManager.checkSecurityAccess(SecurityManager.java:1531) ~[?:?]
at java.security.Provider.check(Provider.java:816) ~[?:?]
at java.security.Provider.remove(Provider.java:523) ~[?:?]
at org.bouncycastle.jce.provider.BouncyCastleProvider.getService(Unknown Source) ~[bcprov-jdk15to18-1.75.jar:1.75.0.0]
at sun.security.jca.ProviderList$ServiceList.tryGet(ProviderList.java:518) ~[?:?]
at sun.security.jca.ProviderList$ServiceList$1.hasNext(ProviderList.java:567) ~[?:?]
at java.security.KeyFactory.nextSpi(KeyFactory.java:312) ~[?:?]
at java.security.KeyFactory.generatePrivate(KeyFactory.java:394) ~[?:?]
at org.opensearch.security.support.PemKeyReader.getPrivateKeyFromByteBuffer(PemKeyReader.java:128) ~[opensearch-security-2.9.0.0.jar:2.9.0.0]
at org.opensearch.security.support.PemKeyReader.toPrivateKey(PemKeyReader.java:109) ~[opensearch-security-2.9.0.0.jar:2.9.0.0]
at org.opensearch.security.support.PemKeyReader.loadKeyFromFile(PemKeyReader.java:197) ~[opensearch-security-2.9.0.0.jar:2.9.0.0]
at com.amazon.dlic.util.SettingsBasedSSLConfigurator.initFromPem(SettingsBasedSSLConfigurator.java:294) ~[opensearch-security-2.9.0.0.jar:2.9.0.0]
at com.amazon.dlic.util.SettingsBasedSSLConfigurator.configureWithSettings(SettingsBasedSSLConfigurator.java:192) ~[opensearch-security-2.9.0.0.jar:2.9.0.0]
at com.amazon.dlic.util.SettingsBasedSSLConfigurator.buildSSLContext(SettingsBasedSSLConfigurator.java:115) ~[opensearch-security-2.9.0.0.jar:2.9.0.0]
at com.amazon.dlic.util.SettingsBasedSSLConfigurator.buildSSLConfig(SettingsBasedSSLConfigurator.java:129) ~[opensearch-security-2.9.0.0.jar:2.9.0.0]
at com.amazon.dlic.auth.http.jwt.keybyoidc.HTTPJwtKeyByOpenIdConnectAuthenticator.getSSLConfig(HTTPJwtKeyByOpenIdConnectAuthenticator.java:65) ~[opensearch-security-2.9.0.0.jar:2.9.0.0]
at com.amazon.dlic.auth.http.jwt.keybyoidc.HTTPJwtKeyByOpenIdConnectAuthenticator.initKeyProvider(HTTPJwtKeyByOpenIdConnectAuthenticator.java:47) ~[opensearch-security-2.9.0.0.jar:2.9.0.0]
at com.amazon.dlic.auth.http.jwt.AbstractHTTPJwtAuthenticator.<init>(AbstractHTTPJwtAuthenticator.java:75) ~[opensearch-security-2.9.0.0.jar:2.9.0.0]
at com.amazon.dlic.auth.http.jwt.keybyoidc.HTTPJwtKeyByOpenIdConnectAuthenticator.<init>(HTTPJwtKeyByOpenIdConnectAuthenticator.java:26) ~[opensearch-security-2.9.0.0.jar:2.9.0.0]
at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
at org.opensearch.security.support.ReflectionHelper.instantiateAAA(ReflectionHelper.java:62) ~[opensearch-security-2.9.0.0.jar:2.9.0.0]
... 25 more
The file permissions should be correct, the owner is opensearch, and I executed chmod 600 on them, as the opensearch warning logs suggested at startup.
What’s weird is if I uncomment the pemcert_filepath, and pemkey_filepath settings on one OpenSearch node (there is 3 in the cluster) in the security config, then run securityadmin.sh on that node, the issue goese away and I can log in, until one of the nodes restarts again.
If I put the settings back after the previous step and rerun securityadmin.sh there is no issue. If I leave them commented and then a node restarts, then the problem appears again, but I can fix it by putting the settings back and running securityadmin.sh again.
Simply running securityadmin.sh never helps. The pemcert_filepath and pemkey_filepath settings need to be changed (commented out/put back) in the config file. Weird behaviour.
Configuration:
_meta:
type: "config"
config_version: 2
config:
dynamic:
authc:
basic_internal_auth_domain:
http_enabled: true
transport_enabled: true
order: 0
http_authenticator:
type: basic
challenge: false
authentication_backend:
type: internal
openid_auth_domain:
http_enabled: true
transport_enabled: true
order: 1
http_authenticator:
type: openid
challenge: false
config:
openid_connect_idp:
enable_ssl: true
verify_hostnames: true
pemtrustedcas_filepath: /usr/share/opensearch/config/certificates/server-ca.crt
pemcert_filepath: /usr/share/opensearch/config/certificates/auth-client.pem
pemkey_filepath: /usr/share/opensearch/config/certificates/auth-client.key
enable_ssl_client_auth: true
subject_key: preferred_username
roles_key: roles
openid_connect_url: https://keycloak-url/auth/realms/asdf/.well-known/openid-configuration
authentication_backend:
type: noop
At this point I’m unable to log out. When I try to log out I get the following message from keycloak: