Hashicorp vault as identity provider 401 unauthorized

Versions (relevant - OpenSearch/Dashboard/Server OS/Browser):

opensearch -2.4.0
opensearch-dashboards - 2.4.0

Describe the issue:

401 unauthorised login to opensearch dashboards with both hashicorp vault and keycloak

Configuration:

Opensearch-master

cat /etc/opensearch/opensearch-security/config.yml
basic_internal_auth_domain:
http_enabled: true
transport_enabled: true
order: 0
http_authenticator:
type: basic
challenge: false
authentication_backend:
type: internal
openid_auth_domain:
http_enabled: true
transport_enabled: true
order: 1
http_authenticator:
type: openid
challenge: false
config:
subject_key: preferred_username
roles_key: roles
openid_connect_url: https://vault.com:8200/v1/sysdev/oidcdemo/identity/oidc/provider/opensearch-oidc-provider/.well-known/openid-configuration
authentication_backend:
type: noop

Opensearch-dashboards:
opensearch.hosts: [“10.x.x.x:8200”]

opensearch.ssl.verificationMode: none

opensearch.username: admin

opensearch.password: admin

opensearch.requestHeadersWhitelist: [authorization, securitytenant]

opensearch_security.multitenancy.enabled: true

opensearch_security.multitenancy.tenants.preferred: [Private, Global]

opensearch_security.readonly_mode.roles: [kibana_read_only]
opensearch_security.cookie.secure: false
opensearch_security.auth.type: openid
opensearch_security.openid.base_redirect_url: “https://os-dev-dashboard
opensearch_security.openid.client_id: “6WuoRNBBXm683tWpJBEbMizjI2hZPgop”
opensearch_security.openid.scope: “openid profile email”
opensearch_security.openid.client_secret: “hvo_secret_uRKFMoG1ZzaBVNMKPB6cPkFRpICtZcaz72BmR252PqTcgVYFGrpQr9Ev8LSm8Mdz”
opensearch_security.openid.connect_url: “http://vault.com:8200/v1/sysdev/oidcdemo/identity/oidc/provider/opensearch-oidc-provider/.well-known/openid-configuration
opensearch_security.openid.verify_hostnames: false

Relevant Logs or Screenshots:

method":“get”,“statusCode”:401,“req”:{“url”:“/auth/openid/login?code=Kbzb5pqhd9e9cuX7MtouwkdoHd9KHh8C&state=1ELFHDJ4UsOSG_cnNvl0oj”,“method”:“get”,“headers”:{“host”:“10.x.x.x:5601”,“connection”:“keep-alive”,“upgrade-insecure-requests”:“1”,“dnt”:“1”,“user-agent”:“Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.52”,“accept”:“text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,/;q=0.8,application/signed-exchange;v=b3;q=0.9”,“accept-encoding”:“gzip, deflate”,“accept-language”:“en-IN,en-GB;q=0.9,en;q=0.8,en-US;q=0.7”},“remoteAddress”:“10.194.59.67”,“userAgent”:“Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.52”},“res”:{“statusCode”:401,“responseTime”:26,“contentLength”:9},“message”:“GET /auth/openid/login?code=Kbzb5pqhd9e9cuX7MtouwkdoHd9KHh8C&state=1ELFHDJ4UsOSG_cnNvl0oj 401 26ms - 9.0B”}

[root@opensearch-master-1 ~]# curl -vk https://os-devl.devops.org.net/_cluster/health -H {“Authorization: Bearer eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJwSlMxQTV2c2pYand2REZqM1NmR1pleldadFEwdnZCMkd4UU5tNlBxSU53In0.eyJleHAiOjE2NjkwOTY0NjIsImlhdCI6MTY2OTA5NjQwMiwianRpIjoiNGUxMDk2ZmItODRkZi00NTZlLTg2ZGItYTAxNzViMjMyZDEyIiwiaXNzIjoiaHR0cDovLzEwLjkxLjE5My4yMjE6ODA4MC9yZWFsbXMvbWFzdGVyIiwiYXVkIjpbIm1hc3Rlci1yZWFsbSIsImFjY291bnQiXSwic3ViIjoiOTQwNDNiMWYtNTlhMy00ZDAxLWJhMDUtZmE1YTE0OGQ0NTAzIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoib3BlbnNlYXJjaC1zc28iLCJzZXNzaW9uX3N0YXRlIjoiOTcyNjUwOWYtM2ZkZi00MmFiLTk1YmMtYzJmMGVmODY1YmY4IiwiYWNyIjoiMSIsInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJjcmVhdGUtcmVhbG0iLCJkZWZhdWx0LXJvbGVzLW1hc3RlciIsIm9mZmxpbmVfYWNjZXNzIiwiYWRtaW4iLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7Im1hc3Rlci1yZWFsbSI6eyJyb2xlcyI6WyJ2aWV3LXJlYWxtIiwidmlldy1pZGVudGl0eS1wcm92aWRlcnMiLCJtYW5hZ2UtaWRlbnRpdHktcHJvdmlkZXJzIiwiaW1wZXJzb25hdGlvbiIsImNyZWF0ZS1jbGllbnQiLCJtYW5hZ2UtdXNlcnMiLCJxdWVyeS1yZWFsbXMiLCJ2aWV3LWF1dGhvcml6YXRpb24iLCJxdWVyeS1jbGllbnRzIiwicXVlcnktdXNlcnMiLCJtYW5hZ2UtZXZlbnRzIiwibWFuYWdlLXJlYWxtIiwidmlldy1ldmVudHMiLCJ2aWV3LXVzZXJzIiwidmlldy1jbGllbnRzIiwibWFuYWdlLWF1dGhvcml6YXRpb24iLCJtYW5hZ2UtY2xpZW50cyIsInF1ZXJ5LWdyb3VwcyJdfSwiYWNjb3VudCI6eyJyb2xlcyI6WyJtYW5hZ2UtYWNjb3VudCIsIm1hbmFnZS1hY2NvdW50LWxpbmtzIiwidmlldy1wcm9maWxlIl19fSwic2NvcGUiOiJvcGVuaWQgcHJvZmlsZSBlbWFpbCIsInNpZCI6Ijk3MjY1MDlmLTNmZGYtNDJhYi05NWJjLWMyZjBlZjg2NWJmOCIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhZG1pbiIsImVtYWlsIjoia2FydGhpazQ5OUBob3RtYWlsLmNvbSJ9.hrD8F9C46hvptJLPJni6e9ddRLPXiGTpjDa4_w6omTAMKF4uaekAExTFqwF0esOjWzXnhnLRXdUQJpVYNj4c5MJ-yn2jAYrwHtZCO9MXNuaPCMvxnTqaHl_UzlUfV6RqHGKBwsH4e262XhSLjIy3IElg81fN-QU2fKB7N4_QOZ5sQDrJhwxnJ7TTTt2Y5s8ORWKI0iQZd_tk8xY-btoMKupg-dt1-ea-pX6GYUiCxScbAyecYlEXtATjqy9NWxEfwIpnt9qeN3fBAow7x9QCn1jrLFu8R0d9np6wFzRj0VheR9maw40NvhxTMVjZ0WU6ZuEhloRuysHUH-dRhnzlLA”,“expires_in”:60,“refresh_expires_in”:1800,“refresh_token”:“eyJhbGciOiJIUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI1NmQ4MGRiNi00OTliLTQ2MWItYThjMy01YjYzZGFkMTRiYjMifQ.eyJleHAiOjE2NjkwOTgyMDIsImlhdCI6MTY2OTA5NjQwMiwianRpIjoiMmJjZDNmZjYtMjQ0ZC00ODEzLWFlMGUtMGE1MDY0MzFhMWUxIiwiaXNzIjoiaHR0cDovLzEwLjkxLjE5My4yMjE6ODA4MC9yZWFsbXMvbWFzdGVyIiwiYXVkIjoiaHR0cDovLzEwLjkxLjE5My4yMjE6ODA4MC9yZWFsbXMvbWFzdGVyIiwic3ViIjoiOTQwNDNiMWYtNTlhMy00ZDAxLWJhMDUtZmE1YTE0OGQ0NTAzIiwidHlwIjoiUmVmcmVzaCIsImF6cCI6Im9wZW5zZWFyY2gtc3NvIiwic2Vzc2lvbl9zdGF0ZSI6Ijk3MjY1MDlmLTNmZGYtNDJhYi05NWJjLWMyZjBlZjg2NWJmOCIsInNjb3BlIjoib3BlbmlkIHByb2ZpbGUgZW1haWwiLCJzaWQiOiI5NzI2NTA5Zi0zZmRmLTQyYWItOTViYy1jMmYwZWY4NjViZjgifQ.u4ncMTRPPKiZUF-X_vrxv13WZCh_x8mw547f60HbGwk”,“token_type”:“Bearer”,“id_token”:“eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJwSlMxQTV2c2pYand2REZqM1NmR1pleldadFEwdnZCMkd4UU5tNlBxSU53In0.eyJleHAiOjE2NjkwOTY0NjIsImlhdCI6MTY2OTA5NjQwMiwiYXV0aF90aW1lIjowLCJqdGkiOiJlZjQxMTRjNi1mMThjLTQ1MTctOTc5NC0xODJiYzk1MGM3ZmYiLCJpc3MiOiJodHRwOi8vMTAuOTEuMTkzLjIyMTo4MDgwL3JlYWxtcy9tYXN0ZXIiLCJhdWQiOiJvcGVuc2VhcmNoLXNzbyIsInN1YiI6Ijk0MDQzYjFmLTU5YTMtNGQwMS1iYTA1LWZhNWExNDhkNDUwMyIsInR5cCI6IklEIiwiYXpwIjoib3BlbnNlYXJjaC1zc28iLCJzZXNzaW9uX3N0YXRlIjoiOTcyNjUwOWYtM2ZkZi00MmFiLTk1YmMtYzJmMGVmODY1YmY4IiwiYXRfaGFzaCI6Ik5XV2tpazE3SXBxNzBpS3phVkhQeEEiLCJhY3IiOiIxIiwic2lkIjoiOTcyNjUwOWYtM2ZkZi00MmFiLTk1YmMtYzJmMGVmODY1YmY4IiwiZW1haWxfdmVyaWZpZWQiOnRydWUsInByZWZlcnJlZF91c2VybmFtZSI6ImFkbWluIiwiZW1haWwiOiJrYXJ0aGlrNDk5QGhvdG1haWwuY29tIn0.eNMmCAl0oLt5z7OtVFulYiSF7aJKQilUpdA3MszsrdJMq7axKrh4qxaswnorZqdXn6Vw6ZxVBY0UeTifB1AwEWaM0kcf02dEZDkX5TALosE6J7MYswDkpgqn5xOctIqOMLlPDwZKUAlv-SNr_aV47kzlBjiS4dK8uRCw1x77kmnSFaMZeHvYSdXIhk_BZ21tgTA741dPt0QZACN8t5MhDySRh420oQ6IRIwaSC_vzbSesdof8kPG05Bc5OTwC09cgTaEOsi3JZmjX30lq0blx0Nb_nM9j6aaD3P8C-TQG5vayaDDAuXfVM37M4wnkenoNNBtRNv2tUPCtOhEwNPvpA”}

  • About to connect() to os-devl.devops.org.net port 443 (#0)

  • Trying 10.44.34.33…

  • Connected to os-devl.devops.org.net (10.44.34.33) port 443 (#0)

  • Initializing NSS with certpath: sql:/etc/pki/nssdb

  • skipping SSL peer certificate verification

  • SSL connection using TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384

  • Server certificate:

  • subject: CN=*.devops.org.net,O=ORG Inc.

  • start date: Jan 12 16:19:21 2022 GMT

  • expire date: Jan 21 16:19:21 2023 GMT

  • common name: *.devops.org.net

  • issuer: CN=Entrust Certification Authority - L1K,OU=“(c) 2012 Entrust, Inc. - for authorized use only”,OU=See www.entrust.net/legal-terms,O=“Entrust, Inc.”,C=US

GET /_cluster/health HTTP/1.1

User-Agent: curl/7.29.0

Host: os-devl.devops.org.net

Accept: /

Authorization: Bearer eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJwSlMxQTV2c2pYand2REZqM1NmR1pleldadFEwdnZCMkd4UU5tNlBxSU53In0.eyJleHAiOjE2NjkwOTY0NjIsImlhdCI6MTY2OTA5NjQwMiwianRpIjoiNGUxMDk2ZmItODRkZi00NTZlLTg2ZGItYTAxNzViMjMyZDEyIiwiaXNzIjoiaHR0cDovLzEwLjkxLjE5My4yMjE6ODA4MC9yZWFsbXMvbWFzdGVyIiwiYXVkIjpbIm1hc3Rlci1yZWFsbSIsImFjY291bnQiXSwic3ViIjoiOTQwNDNiMWYtNTlhMy00ZDAxLWJhMDUtZmE1YTE0OGQ0NTAzIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoib3BlbnNlYXJjaC1zc28iLCJzZXNzaW9uX3N0YXRlIjoiOTcyNjUwOWYtM2ZkZi00MmFiLTk1YmMtYzJmMGVmODY1YmY4IiwiYWNyIjoiMSIsInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJjcmVhdGUtcmVhbG0iLCJkZWZhdWx0LXJvbGVzLW1hc3RlciIsIm9mZmxpbmVfYWNjZXNzIiwiYWRtaW4iLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7Im1hc3Rlci1yZWFsbSI6eyJyb2xlcyI6WyJ2aWV3LXJlYWxtIiwidmlldy1pZGVudGl0eS1wcm92aWRlcnMiLCJtYW5hZ2UtaWRlbnRpdHktcHJvdmlkZXJzIiwiaW1wZXJzb25hdGlvbiIsImNyZWF0ZS1jbGllbnQiLCJtYW5hZ2UtdXNlcnMiLCJxdWVyeS1yZWFsbXMiLCJ2aWV3LWF1dGhvcml6YXRpb24iLCJxdWVyeS1jbGllbnRzIiwicXVlcnktdXNlcnMiLCJtYW5hZ2UtZXZlbnRzIiwibWFuYWdlLXJlYWxtIiwidmlldy1ldmVudHMiLCJ2aWV3LXVzZXJzIiwidmlldy1jbGllbnRzIiwibWFuYWdlLWF1dGhvcml6YXRpb24iLCJtYW5hZ2UtY2xpZW50cyIsInF1ZXJ5LWdyb3VwcyJdfSwiYWNjb3VudCI6eyJyb2xlcyI6WyJtYW5hZ2UtYWNjb3VudCIsIm1hbmFnZS1hY2NvdW50LWxpbmtzIiwidmlldy1wcm9maWxlIl19fSwic2NvcGUiOiJvcGVuaWQgcHJvZmlsZSBlbWFpbCIsInNpZCI6Ijk3MjY1MDlmLTNmZGYtNDJhYi05NWJjLWMyZjBlZjg2NWJmOCIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhZG1pbiIsImVtYWlsIjoia2FydGhpazQ5OUBob3RtYWlsLmNvbSJ9.hrD8F9C46hvptJLPJni6e9ddRLPXiGTpjDa4_w6omTAMKF4uaekAExTFqwF0esOjWzXnhnLRXdUQJpVYNj4c5MJ-yn2jAYrwHtZCO9MXNuaPCMvxnTqaHl_UzlUfV6RqHGKBwsH4e262XhSLjIy3IElg81fN-QU2fKB7N4_QOZ5sQDrJhwxnJ7TTTt2Y5s8ORWKI0iQZd_tk8xY-btoMKupg-dt1-ea-pX6GYUiCxScbAyecYlEXtATjqy9NWxEfwIpnt9qeN3fBAow7x9QCn1jrLFu8R0d9np6wFzRj0VheR9maw40NvhxTMVjZ0WU6ZuEhloRuysHUH-dRhnzlLA

< HTTP/1.1 401 Unauthorized

< WWW-Authenticate: Basic realm=“OpenSearch Security”

< content-type: text/plain; charset=UTF-8

< content-length: 12