Internal server error, statusCode: 500, after Keycloak openid login

Security
1

Versions:
OpenSearch 2.4, OpenSearch Dashboards 2.4, Keycloak 20.0.1 all running in docker

Describe the issue:
After logging in to OpenSearch Dashboards through keycloak I get:
statusCode: 500
error: “Internal Server Error”
message: “An internal server error occured.”

Relevant Logs:
OpenSearch:

[2023-07-20T11:05:39,418][WARN ][o.o.s.h.HTTPBasicAuthenticator] [node1] No 'Basic Authorization' header, send 401 and 'WWW-Authenticate Basic'
[2023-07-20T14:18:40,709][WARN ][o.o.s.h.HTTPBasicAuthenticator] [node1] No 'Basic Authorization' header, send 401 and 'WWW-Authenticate Basic'
[2023-07-20T14:18:41,027][WARN ][o.o.s.h.HTTPBasicAuthenticator] [node1] No 'Basic Authorization' header, send 401 and 'WWW-Authenticate Basic'

Dashboards:

{"type":"response","@timestamp":"2023-07-20T12:18:40Z","tags":[],"pid":1,"method":"get","statusCode":302,"req":{"url":"/auth/openid/login","method":"get","headers":{"host":"dashboard-url","user-agent":"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0","accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,
image/webp,*/*;q=0.8","accept-language":"en-US,en;q=0.5","accept-encoding":"gzip, deflate, br","connection":"keep-alive","upgrade-insecure-requests":"1","sec-fetch-dest":"document","sec-fetch
-mode":"navigate","sec-fetch-site":"none","sec-fetch-user":"?1"},"remoteAddress":"10.36.65.17","userAgent":"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0"},"r
es":{"statusCode":302,"responseTime":4,"contentLength":9},"message":"GET /auth/openid/login 302 4ms - 9.0B"}
{"type":"response","@timestamp":"2023-07-20T12:18:40Z","tags":[],"pid":1,"method":"get","statusCode":302,"req":{"url":"/auth/openid/login?state=DCkYnBg5K-LyzcZuuiduU9&session_state=a6d69df5-d
5bd-4e98-a1b5-b94c2d4ad8ee&code=9c738992-bbfc-411e-bf31-d42706679aab.a6d69df5-d5bd-4e98-a1b5-b94c2d4ad8ee.d021f693-b83c-45bf-aeaf-f7f624b96564","method":"get","headers":{"host":"dashboard-jer
-bm-01a.sna.internal:5669","user-agent":"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0","accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/
avif,image/webp,*/*;q=0.8","accept-language":"en-US,en;q=0.5","accept-encoding":"gzip, deflate, br","connection":"keep-alive","upgrade-insecure-requests":"1","sec-fetch-dest":"document","sec-
fetch-mode":"navigate","sec-fetch-site":"none","sec-fetch-user":"?1"},"remoteAddress":"10.36.65.17","userAgent":"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0
"},"res":{"statusCode":302,"responseTime":36,"contentLength":9},"message":"GET /auth/openid/login?state=DCkYnBg5K-LyzcZuuiduU9&session_state=a6d69df5-d5bd-4e98-a1b5-b94c2d4ad8ee&code=9c738992
-bbfc-411e-bf31-d42706679aab.a6d69df5-d5bd-4e98-a1b5-b94c2d4ad8ee.d021f693-b83c-45bf-aeaf-f7f624b96564 302 36ms - 9.0B"}
{"type":"log","@timestamp":"2023-07-20T12:18:40Z","tags":["error","plugins","securityDashboards"],"pid":1,"message":"Failed to resolve user tenant: Error: Failed authentication: Authenticatio
n Exception"}
{"type":"response","@timestamp":"2023-07-20T12:18:40Z","tags":[],"pid":1,"method":"get","statusCode":302,"req":{"url":"/","method":"get","headers":{"host":"dashboard-url","user-agent":"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0","accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=
0.8","accept-language":"en-US,en;q=0.5","accept-encoding":"gzip, deflate, br","connection":"keep-alive","upgrade-insecure-requests":"1","sec-fetch-dest":"document","sec-fetch-mode":"navigate"
,"sec-fetch-site":"none","sec-fetch-user":"?1"},"remoteAddress":"10.36.65.17","userAgent":"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0"},"res":{"statusCode"
:302,"responseTime":65,"contentLength":9},"message":"GET / 302 65ms - 9.0B"}
{"type":"response","@timestamp":"2023-07-20T12:18:40Z","tags":[],"pid":1,"method":"get","statusCode":302,"req":{"url":"/auth/openid/login","method":"get","headers":{"host":"dashboard-url","user-agent":"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0","accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,
image/webp,*/*;q=0.8","accept-language":"en-US,en;q=0.5","accept-encoding":"gzip, deflate, br","connection":"keep-alive","upgrade-insecure-requests":"1","sec-fetch-dest":"document","sec-fetch
-mode":"navigate","sec-fetch-site":"none","sec-fetch-user":"?1"},"remoteAddress":"10.36.65.17","userAgent":"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0"},"r
es":{"statusCode":302,"responseTime":3,"contentLength":9},"message":"GET /auth/openid/login 302 3ms - 9.0B"}
{"type":"response","@timestamp":"2023-07-20T12:18:40Z","tags":[],"pid":1,"method":"get","statusCode":302,"req":{"url":"/auth/openid/login?state=gGvkT2fsdZcPEZSpsGU0kF&session_state=a6d69df5-d
5bd-4e98-a1b5-b94c2d4ad8ee&code=718410bd-0ca3-4c95-92bf-20e66ac3faa5.a6d69df5-d5bd-4e98-a1b5-b94c2d4ad8ee.d021f693-b83c-45bf-aeaf-f7f624b96564","method":"get","headers":{"host":"dashboard-jer
-bm-01a.sna.internal:5669","user-agent":"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0","accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/
avif,image/webp,*/*;q=0.8","accept-language":"en-US,en;q=0.5","accept-encoding":"gzip, deflate, br","connection":"keep-alive","upgrade-insecure-requests":"1","sec-fetch-dest":"document","sec-
fetch-mode":"navigate","sec-fetch-site":"none","sec-fetch-user":"?1"},"remoteAddress":"10.36.65.17","userAgent":"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0
"},"res":{"statusCode":302,"responseTime":33,"contentLength":9},"message":"GET /auth/openid/login?state=gGvkT2fsdZcPEZSpsGU0kF&session_state=a6d69df5-d5bd-4e98-a1b5-b94c2d4ad8ee&code=718410bd
-0ca3-4c95-92bf-20e66ac3faa5.a6d69df5-d5bd-4e98-a1b5-b94c2d4ad8ee.d021f693-b83c-45bf-aeaf-f7f624b96564 302 33ms - 9.0B"}
{"type":"log","@timestamp":"2023-07-20T12:18:41Z","tags":["error","http","server","OpenSearchDashboards"],"pid":1,"message":"Error: Authentication Exception\n    at SecurityClient.authinfo (/
usr/share/opensearch-dashboards/plugins/securityDashboards/server/backend/opensearch_security_client.ts:115:13)\n    at processTicksAndRejections (internal/process/task_queues.js:95:5)\n    a
t /usr/share/opensearch-dashboards/plugins/securityDashboards/server/auth/types/authentication_type.ts:208:18\n    at Object.interceptAuth [as authenticate] (/usr/share/opensearch-dashboards/
src/core/server/http/lifecycle/auth.js:112:22)\n    at exports.Manager.execute (/usr/share/opensearch-dashboards/node_modules/@hapi/hapi/lib/toolkit.js:60:28)\n    at module.exports.internals
.Auth._authenticate (/usr/share/opensearch-dashboards/node_modules/@hapi/hapi/lib/auth.js:273:30)\n    at Request._lifecycle (/usr/share/opensearch-dashboards/node_modules/@hapi/hapi/lib/requ
est.js:371:32)\n    at Request._execute (/usr/share/opensearch-dashboards/node_modules/@hapi/hapi/lib/request.js:281:9)"}
{"type":"error","@timestamp":"2023-07-20T12:18:41Z","tags":[],"pid":1,"level":"error","error":{"message":"Internal Server Error","name":"Error","stack":"Error: Internal Server Error\n    at H
apiResponseAdapter.toInternalError (/usr/share/opensearch-dashboards/src/core/server/http/router/response_adapter.js:80:19)\n    at Object.interceptAuth [as authenticate] (/usr/share/opensear
ch-dashboards/src/core/server/http/lifecycle/auth.js:151:34)\n    at processTicksAndRejections (internal/process/task_queues.js:95:5)\n    at exports.Manager.execute (/usr/share/opensearch-da
shboards/node_modules/@hapi/hapi/lib/toolkit.js:60:28)\n    at module.exports.internals.Auth._authenticate (/usr/share/opensearch-dashboards/node_modules/@hapi/hapi/lib/auth.js:273:30)\n    a
t Request._lifecycle (/usr/share/opensearch-dashboards/node_modules/@hapi/hapi/lib/request.js:371:32)\n    at Request._execute (/usr/share/opensearch-dashboards/node_modules/@hapi/hapi/lib/re
quest.js:281:9)"},"url":"https://dashboard-url","message":"Internal Server Error"}
{"type":"response","@timestamp":"2023-07-20T12:18:41Z","tags":[],"pid":1,"method":"get","statusCode":500,"req":{"url":"/","method":"get","headers":{"host":"dashboard-url","user-agent":"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0","accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=
0.8","accept-language":"en-US,en;q=0.5","accept-encoding":"gzip, deflate, br","connection":"keep-alive","upgrade-insecure-requests":"1","sec-fetch-dest":"document","sec-fetch-mode":"navigate"
,"sec-fetch-site":"none","sec-fetch-user":"?1"},"remoteAddress":"10.36.65.17","userAgent":"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0"},"res":{"statusCode"
:500,"responseTime":35,"contentLength":9},"message":"GET / 500 35ms - 9.0B"}

Keycloak:

2023-07-13 15:24:53,461 WARN  [org.keycloak.events] (executor-thread-129) type=LOGIN_ERROR, realmId=myrealm, clientId=null, userId=null, ipAddress=..., error=cookie_not_found

HTTPS is in use for keycloak and opensearch with self signed certs.
I could not figure out what the root of the problem is.

config.yml:

_meta:
  type: "config"
  config_version: 2
config:
  dynamic:
    authc:
      basic_internal_auth_domain:
        http_enabled: true
        transport_enabled: true
        order: 0
        http_authenticator:
          type: basic
          challenge: false
        authentication_backend:
          type: internal
      openid_auth_domain:
        http_enabled: true
        transport_enabled: true
        order: 1
        http_authenticator:
          type: openid
          challenge: false
          config:
            openid_connect_idp:
              enable_ssl: true
              verify_hostnames: true
              pemtrustedcas_filepath: /usr/share/opensearch/config/certificates/server-ca.crt
              pemcert_filepath: /usr/share/opensearch/config/certificates/auth-client.pem
              pemkey_filepath: /usr/share/opensearch/config/certificates/auth-client.key
              enable_ssl_client_auth: true
            subject_key: preferred_username
            roles_key: roles
            openid_connect_url: https://keycloak-url/auth/realms/myrealm/.well-known/openid-configuration
        authentication_backend:
          type: noop

opensearch_dashboards.yml:

server.name: dashboards
server.host: "0.0.0.0"

opensearch.username: kibanaserver
opensearch.password: kibanaserver

server.ssl.enabled: true
server.ssl.certificate: /usr/share/opensearch-dashboards/config/certificates/dashboard.pem
server.ssl.key: /usr/share/opensearch-dashboards/config/certificates/dashboard.key

opensearch.ssl.certificateAuthorities:
          [/usr/share/opensearch-dashboards/config/certificates/server-ca.crt]
opensearch.ssl.verificationMode: full

opensearch_security.cookie.secure: true

opensearch.requestHeadersAllowlist: ["securitytenant", "Authorization"]
opensearch_security.multitenancy.enabled: true
opensearch_security.multitenancy.tenants.enable_global: true
opensearch_security.multitenancy.tenants.enable_private: true
opensearch_security.multitenancy.tenants.preferred: ["Global"]
opensearch_security.multitenancy.enable_filter: false

opensearch_security.auth.type: openid
opensearch_security.openid.connect_url: https://keycloak-url/auth/realms/myrealm/.well-known/openid-configuration
opensearch_security.openid.base_redirect_url: 'https://dashboard-url:5669' # docker published port
opensearch_security.openid.client_id: opensearch-client
opensearch_security.openid.client_secret: secret
opensearch_security.openid.root_ca: /usr/share/opensearch-dashboards/config/certificates/server-ca.crt
opensearch_security.openid.verify_hostnames: "true"
2

@szurokl Please share config.yml and opensearch_dashboards.yml files.

3

Hi @pablo

I updated the post with the content of these config files.

4

Is server-ca.crt a Kibana certificate or Keycloak?

You don’t need this in the config.yml as pemtrustedcas_filepath is enough.

5

It’s a root ca, which signed both keycloak cert and opensearch cert

6

@szurokl I can’t find opensearch.hosts option. Could you share your docker-compose.yml file?

7

Sorry for the late response, here is the compose file and opensearch.hosts for dashboards:

opensearch:
    mem_limit: 6g
    user: 1000:1000
    image: opensearchproject/opensearch:2.4.0
    container_name: opensearch
    env_file: ./config/opensearch/.env
    networks:
      - mynetwork
    ports:
      - x.x.x.x:9200:9200
      - x.x.x.x:9300:9300
    extra_hosts: ['keycloak-url:x.x.x.x','ops-node-1:x.x.x.x','ops-node-2:x.x.x.x','ops-node-3:x.x.x.x']
    volumes:
      - ./data/opensearch:/usr/share/opensearch/data
      - ./config/opensearch/opensearch.yml:/usr/share/opensearch/config/opensearch.yml
      - ./config/opensearch/security-config.yml:/usr/share/opensearch/config/opensearch-security/config.yml
      - ./config/opensearch/internal_users.yml:/usr/share/opensearch/config/opensearch-security/internal_users.yml
      - ./config/opensearch/roles.yml:/usr/share/opensearch/config/opensearch-security/roles.yml
      - ./config/opensearch/roles_mapping.yml:/usr/share/opensearch/config/opensearch-security/roles_mapping.yml
      - ./config/opensearch/certs:/usr/share/opensearch/config/certificates
      - /path/to/server-ca.crt:/usr/share/opensearch/config/certificates/server-ca.crt
      - /usr/local/lib/opentelemetry-javaagent.jar:/app/opentelemetry-javaagent.jar
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 65536
        hard: 65536
    restart: unless-stopped
    logging:
      driver: journald
      options:
        tag: opensearch

  dashboards:
    mem_limit: 1g
    user: 1000:1000
    image: opensearchproject/opensearch-dashboards:2.4.0
    container_name: dashboards
    env_file: ./config/dashboards/.env
    networks:
      - mynetwork
    ports:
      - x.x.x.x:5669:5601
    extra_hosts: [''keycloak-url:x.x.x.x','ops-node-1:x.x.x.x','ops-node-2:x.x.x.x','ops-node-3:x.x.x.x']
    volumes:
      - ./config/dashboards/dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml
      - ./config/dashboards/certs:/usr/share/opensearch-dashboards/config/certificates
      - /path/to/server-ca.crt:/usr/share/opensearch-dashboards/config/certificates/server-ca.crt
    restart: unless-stopped
    logging:
      driver: journald
      options:
        tag: dashboards

./config/dashboards/.env:

OPENSEARCH_HOSTS='["https://ops-node-1:9200","https://ops-node-2:9200","https://ops-node-3:9200"]'
DISABLE_INSTALL_DEMO_CONFIG=true