Versions (relevant - OpenSearch/Dashboard/Server OS/Browser):
Opensearch 2.8.0 (issue first seen in 2.7.0. see below)
Keycloak 21.1.1
Running in k8s environment
Describe the issue:
We recently upgraded Opensearch from 2.4.1 to 2.8.0. We use Keycloack as our OpenID authentication. We have Keylcoak configured to expire JWT access token in 5 min. When a user stays idle on a dashboard for 5 min, the next action that is performed, an error is raised. Then the subsequent action that is performed is successful. This was not witnessed in 2.4.1 as it seems that the access token is being refreshed. I have not been able to prove this so this is just my guess as to the root issue. I tried installing previous versions to see when this issue actually was present and it looks like this starts in 2.7.0. Was there a change that may have broken the handling of expired access tokens?
This is the error that is seen in the Opensearch logs:
Extracting JWT token from eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJERWk5T0lMdXljV3k0TVg1cmxFS0V3UzhvR3NjVjQwUlJwSVN6LXBJN0xNIn0.eyJleHAiOjE2OTA4MTIyNDYsImlhdCI6MTY5MDgxMTk0NiwiYXV0aF90aW1lIjoxNjkwODEwOTk2LCJqdGkiOiI5NDRjNjY2NC01YjkwLTQ2ZDUtOWUyNC0zODJkZGQ4MzVmNGUiLCJpc3MiOiJodHRwczovLzEwLjIuMjcuODEvcmVhbG1zL1RpdGFuaXVtIiwiYXVkIjoidGVsZW1ldHJ5Iiwic3ViIjoiMDA2YjUxZTItMDJiNy00YTE1LTk4ZGQtNmQ2NjM2MzMyNWZjIiwidHlwIjoiSUQiLCJhenAiOiJ0ZWxlbWV0cnkiLCJzZXNzaW9uX3N0YXRlIjoiNTU4Y2Y5YzUtNGNjOC00YWI4LWEwZTItZWJjOTVlMTQ5MTMxIiwiYXRfaGFzaCI6Im5BZFQ5cGpCQm5ZNmJ2NkZka29nX0EiLCJzaWQiOiI1NThjZjljNS00Y2M4LTRhYjgtYTBlMi1lYmM5NWUxNDkxMzEiLCJhZGRyZXNzIjp7fSwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJuIG4iLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJubmFkbWluIiwiZ2l2ZW5fbmFtZSI6Im4iLCJmYW1pbHlfbmFtZSI6Im4iLCJ0aXRhbl9pdW1fcm9sZXMiOlsiKnx8dGVsZW1ldHJ5LXVpfHx1aXx8V1JJVEUiLCIqfHwqfHwqfHxXUklURSIsIm9mZmxpbmVfYWNjZXNzIiwiZGVmYXVsdC1yb2xlcy10aXRhbml1bSIsInVtYV9hdXRob3JpemF0aW9uIl0sImVtYWlsIjoibm5hZG1pbkB0aXRhbml1bXBsYXRmb3JtLmNvbSJ9.aYL8-W1zO0ogOH3kGxxc4eJQJg-hWJ4KJ9hIPfDyCi27ANKcvkOX8jBQDyN-sBlah9rDVEn2yI-i9kRewmer8D2HlWNEpdRW2Fx6e4DOQqE9zWl7Oo-Ef4bbLNYD7Jg9bF6cCNe6iyONMajykQ1Ht-WXRq_ljTfspncU4z-Thb_JD91tGxS_PIjuzUxNluHBu_8UTEMLUQcacwwyw9GVciHAVluBljDFeduVXYrPCOMTePWPmo1dsXee5fUl1NnEuyZARt4tBbcaCZ0ZjhsMt4utiSdBbn5-ZFABPlSmQoMdhTRO7oi3XM_x7waAlXHhSv9yHBOlsANQxU2Xl3uryg failed
com.amazon.dlic.auth.http.jwt.keybyoidc.BadCredentialsException: The token has expired
at com.amazon.dlic.auth.http.jwt.keybyoidc.JwtVerifier.getVerifiedJwtToken(JwtVerifier.java:83) ~[opensearch-security-2.8.0.0.jar:2.8.0.0]
at com.amazon.dlic.auth.http.jwt.AbstractHTTPJwtAuthenticator.extractCredentials0(AbstractHTTPJwtAuthenticator.java:115) [opensearch-security-2.8.0.0.jar:2.8.0.0]
at com.amazon.dlic.auth.http.jwt.AbstractHTTPJwtAuthenticator$1.run(AbstractHTTPJwtAuthenticator.java:97) [opensearch-security-2.8.0.0.jar:2.8.0.0]
at com.amazon.dlic.auth.http.jwt.AbstractHTTPJwtAuthenticator$1.run(AbstractHTTPJwtAuthenticator.java:94) [opensearch-security-2.8.0.0.jar:2.8.0.0]
at java.security.AccessController.doPrivileged(AccessController.java:318) [?:?]
at com.amazon.dlic.auth.http.jwt.AbstractHTTPJwtAuthenticator.extractCredentials(AbstractHTTPJwtAuthenticator.java:94) [opensearch-security-2.8.0.0.jar:2.8.0.0]
at org.opensearch.security.auth.BackendRegistry.authenticate(BackendRegistry.java:244) [opensearch-security-2.8.0.0.jar:2.8.0.0]
at org.opensearch.security.filter.SecurityRestFilter.checkAndAuthenticateRequest(SecurityRestFilter.java:191) [opensearch-security-2.8.0.0.jar:2.8.0.0]
at org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:124) [opensearch-security-2.8.0.0.jar:2.8.0.0]
at org.opensearch.rest.RestController.dispatchRequest(RestController.java:320) [opensearch-2.8.0.jar:2.8.0]
at org.opensearch.rest.RestController.tryAllHandlers(RestController.java:411) [opensearch-2.8.0.jar:2.8.0]
at org.opensearch.rest.RestController.dispatchRequest(RestController.java:249) [opensearch-2.8.0.jar:2.8.0]
at org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) [opensearch-security-2.8.0.0.jar:2.8.0.0]
at org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:366) [opensearch-2.8.0.jar:2.8.0]
at org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:445) [opensearch-2.8.0.jar:2.8.0]
at org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:356) [opensearch-2.8.0.jar:2.8.0]
at org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) [transport-netty4-client-2.8.0.jar:2.8.0]
at org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) [transport-netty4-client-2.8.0.jar:2.8.0]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) [transport-netty4-client-2.8.0.jar:2.8.0]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) [netty-codec-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) [netty-codec-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) [netty-codec-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.91.Final.jar:4.1.91.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) [netty-handler-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) [netty-codec-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) [netty-handler-4.1.91.Final.jar:4.1.91.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) [netty-handler-4.1.91.Final.jar:4.1.91.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) [netty-handler-4.1.91.Final.jar:4.1.91.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) [netty-codec-4.1.91.Final.jar:4.1.91.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) [netty-codec-4.1.91.Final.jar:4.1.91.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) [netty-codec-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:689) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:652) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.91.Final.jar:4.1.91.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.91.Final.jar:4.1.91.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.91.Final.jar:4.1.91.Final]
at java.lang.Thread.run(Thread.java:833) [?:?]
Caused by: org.apache.cxf.rs.security.jose.jwt.JwtException: The token has expired
at org.apache.cxf.rs.security.jose.jwt.JwtUtils.validateJwtExpiry(JwtUtils.java:58) ~[cxf-rt-rs-security-jose-3.5.5.jar:3.5.5]
at com.amazon.dlic.auth.http.jwt.keybyoidc.JwtVerifier.validateClaims(JwtVerifier.java:117) ~[opensearch-security-2.8.0.0.jar:2.8.0.0]
at com.amazon.dlic.auth.http.jwt.keybyoidc.JwtVerifier.getVerifiedJwtToken(JwtVerifier.java:79) ~[opensearch-security-2.8.0.0.jar:2.8.0.0]
... 71 more
[2023-07-31T14:06:13,091][WARN ][o.o.s.a.BackendRegistry ] [titan-ium-telemetry-db-1.titan-ium-telemetry-db-headless.signaling-firewall.svc.cluster.local] Authentication finally failed for null from 10.233.127.207:54812
Configuration:
opensearch-dashboards.yml:
opensearch:
hosts:
- https://titan-ium-telemetry-db.signaling-firewall.svc.cluster.local:9200
username: "${TELEMETRY_USERNAME}"
password: "${TELEMETRY_PASSWORD}"
ssl:
truststore:
path: /secure/TITAN.IUM/keystore/keystore
password: "${NTI_KEYSTORE_PASSWORD}"
requestHeadersAllowlist:
- Authorization
- securitytenant
# Including old header name for backwards compatability
- security_tenant
server:
host: 0.0.0.0
ssl:
enabled: true
keystore:
path: /secure/TITAN.IUM/keystore/keystore
password: "${NTI_KEYSTORE_PASSWORD}"
opensearch_security:
auth:
type: openid
cookie:
secure: true
password: "${TELEMETRY_COOKIE_SECRET}"
openid:
connect_url: "https://titan-ium-oauth.signaling-firewall.svc.cluster.local/realms/Titanium/.well-known/openid-configuration/"
client_secret: "${TELEMETRY_CLIENT_SECRET}"
client_id: "${TELEMETRY_CLIENT_ID}"
root_ca: /secure/TITAN.IUM/oauth-ca/oauth-ca.crt
base_redirect_url: https://10.2.27.80/
readonly_mode:
roles:
- kibana_read_only
multitenancy:
enabled: false
# Increase autocomplete timeout and terminate after so controls visualizations
# are as complete as possible
opensearchDashboards:
autocompleteTimeout: 100000
autocompleteTerminateAfter: 10000000
branding:
logo:
defaultUrl: "https://10.2.27.80/ui/favicons/favicon.png"
mark:
defaultUrl: "https://10.2.27.80/ui/favicons/favicon.png"
loadingLogo:
defaultUrl: "https://10.2.27.80/ui/favicons/favicon.png"
faviconUrl: "https://10.2.27.80/ui/favicons/favicon.png"
applicationTitle: "Titan.ium Telemetry"
opensearch.yml
network:
host: ${HOSTNAME}
publish_host: ${HOSTNAME}.titan-ium-telemetry-db-headless.signaling-firewall.svc.cluster.local
cluster:
name: titan-ium-telemetry-db
initial_master_nodes:
- titan-ium-telemetry-db-0.titan-ium-telemetry-db-headless.signaling-firewall.svc.cluster.local
discovery:
seed_hosts:
- titan-ium-telemetry-db-headless.signaling-firewall.svc.cluster.local
node:
name: ${HOSTNAME}.titan-ium-telemetry-db-headless.signaling-firewall.svc.cluster.local
roles:
- cluster_manager
- data
plugins:
security:
ssl:
transport:
enforce_hostname_verification: true
keystore_filepath: keystore/keystore
keystore_password: ${NTI_KEYSTORE_PASSWORD}
truststore_filepath: keystore/keystore
truststore_password: ${NTI_KEYSTORE_PASSWORD}
http:
enabled: true
keystore_filepath: keystore/keystore
keystore_password: ${NTI_KEYSTORE_PASSWORD}
truststore_filepath: keystore/keystore
truststore_password: ${NTI_KEYSTORE_PASSWORD}
# Enable the secuirty init script
allow_default_init_securityindex: true
filter_securityindex_from_all_requests: true
# Tracks login attempts
audit:
type: internal_opensearch
routes:
FAILED_LOGIN:
endpoints:
- default
BAD_HEADERS:
endpoints:
- default
MISSING_PRIVILEGES:
endpoints:
- default
GRANTED_PRIVILEGES:
endpoints:
- default
OPENDISTRO_SECURITY_INDEX_ATTEMPT:
endpoints:
- default
SSL_EXCEPTION:
endpoints:
- default
AUTHENTICATED:
endpoints:
- default
INDEX_EVENT:
endpoints:
- default
COMPLIANCE_DOC_READ:
endpoints:
- default
COMPLIANCE_DOC_WRITE:
endpoints:
- default
COMPLIANCE_EXTERNAL_CONFIG:
endpoints:
- default
COMPLIANCE_INTERNAL_CONFIG_READ:
endpoints:
- default
COMPLIANCE_INTERNAL_CONFIG_WRITE:
endpoints:
- default
restapi:
roles_enabled:
- all_access
- security_rest_api_access
authcz:
admin_dn:
- "CN=titan-ium"
nodes_dn:
- "CN=titan-ium"
# Disable warning message "No 'Basic Authorization' header, send 401 and 'WWW-Authenticate Basic'"
# Occurs when an OpenID Connect user accesses the database
logger.org.opensearch.security.http.HTTPBasicAuthenticator: ERROR
# Workaround for https://github.com/opensearch-project/security/issues/1307
logger.org.opensearch.deprecation.cluster.metadata: ERROR
/opensearch-security/config.yml
_meta:
type: "config"
config_version: 2
config:
dynamic:
kibana:
multitenancy_enabled: false
http:
anonymous_auth_enabled: false
xff:
enabled: false
authc:
basic_internal_auth_domain:
description: "Authenticate via HTTP Basic against internal users database"
http_enabled: true
transport_enabled: true
order: 0
http_authenticator:
type: basic
challenge: false
authentication_backend:
type: internal
openid_auth_domain:
http_enabled: true
transport_enabled: true
order: 1
http_authenticator:
type: openid
challenge: false
config:
openid_connect_idp.enable_ssl: true
openid_connect_idp.verify_hostnames: true
openid_connect_idp.pemtrustedcas_filepath: /usr/share/opensearch/config/oauth-ca/oauth-ca.crt
subject_key: preferred_username
roles_key: titan_ium_roles
openid_connect_url: https://titan-ium-oauth.signaling-firewall.svc.cluster.local/realms/Titanium/.well-known/openid-configuration/
authentication_backend:
type: noop
Relevant Logs or Screenshots: