Versions (relevant - OpenSearch/Dashboard/Server OS/Browser):
Opensearch version used is 2.2.1
works fiine with the 1.3.2 opensearch version
Describe the issue:
After the first log rotation the following log is observed inside the dashboards pod
Opensearch dashboards is installed on kubernetes environment
*
{"log":{"message":"detected rotation of /tmp/mainContainerLogs/dashboards.log; waiting 5 seconds"},"type":"log","level":"unavailable","timezone":"UTC","system":"aryan","systemid":"3363716ae1c447e5b0ca626c6a0bf536","host":"test-11-aryan-dashboards-7797654784-6c6dl.ary-ns-11","time":"2024-02-02T13:13:04+0000"} {"log":{"message":"Reloading logging configuration due to SIGHUP."},"extension":{"type":"log","tags":["info","cli","config"],"pid":116},"type":"log","level":"info","system":"aryan","systemid":"3363716ae1c447e5b0ca626c6a0bf536","host":"test-11-aryan-dashboards-7797654784-6c6dl.ary-ns-11","timezone":"UTC","time":"2024-02-02T13:13:04Z"} {"log":{"message":"following tail of /tmp/mainContainerLogs/dashboards.log"},"type":"log","level":"unavailable","timezone":"UTC","system":"aryan","systemid":"3363716ae1c447e5b0ca626c6a0bf536","host":"test-11-aryan-dashboards-7797654784-6c6dl.ary-ns-11","time":"2024-02-02T13:13:04+0000"} {"time":"2024-02-02T13:13:04+0000","level":"info","message":"following tail of /tmp/mainContainerLogs/dashboards.log","worker_id":0} {"log":{"message":"TypeError: Cannot read property 'reconfigure' of undefined\n at OsdServer.applyLoggingConfiguration (/usr/share/opensearch-dashboards/src/legacy/server/osd_server.js:182:42)\n at TapSubscriber._tapNext (/usr/share/opensearch-dashboards/src/core/server/legacy/legacy_service.js:77:24)\n at TapSubscriber._next (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/operators/tap.js:59:27)\n at TapSubscriber.Subscriber.next (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/Subscriber.js:66:18)\n at CombineLatestSubscriber.notifyNext (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/observable/combineLatest.js:97:34)\n at InnerSubscriber._next (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/InnerSubscriber.js:28:21)\n at InnerSubscriber.Subscriber.next (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/Subscriber.js:66:18)\n at ReplaySubject.Subject.next (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/Subject.js:60:25)\n at ReplaySubject.nextInfiniteTimeWindow (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/ReplaySubject.js:50:31)\n at Object.next (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/operators/shareReplay.js:36:29)\n at SafeSubscriber.__tryOrUnsub (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/Subscriber.js:205:16)\n at SafeSubscriber.next (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/Subscriber.js:143:22)\n at Subscriber._next (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/Subscriber.js:89:26)\n at Subscriber.next (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/Subscriber.js:66:18)\n at MapSubscriber._next (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/operators/map.js:55:26)\n at MapSubscriber.Subscriber.next (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/Subscriber.js:66:18)"},"extension":{"type":"log","tags":["error","legacy-service"],"pid":116},"type":"log","level":"info","system":"aryan","systemid":"3363716ae1c447e5b0ca626c6a0bf536","host":"test-11-aryan-dashboards-7797654784-6c6dl.ary-ns-11","timezone":"UTC","time":"2024-02-02T13:13:04Z"} {"log":{"message":"New logging configuration:\n{\n \"ops\": {\n \"cGroupOverrides\": {\n \"cpuPath\": \"/\",\n \"cpuAcctPath\": \"/\"\n },\n \"interval\": 5000\n },\n \"logging\": {\n \"dest\": \"/tmp/mainContainerLogs/dashboards.log\",\n \"rotate\": {\n \"enabled\": true,\n \"everyBytes\": 1048577,\n \"keepFiles\": 7,\n \"pollingInterval\": 10000,\n \"usePolling\": false\n },\n \"silent\": false,\n \"quiet\": false,\n \"verbose\": false,\n \"events\": {},\n \"filter\": {},\n \"json\": true,\n \"timezone\": \"UTC\"\n }\n}"},"extension":{"type":"log","tags":["info","config"],"pid":116},"type":"log","level":"info","system":"aryan","systemid":"3363716ae1c447e5b0ca626c6a0bf536","host":"test-11-aryan-dashboards-7797654784-6c6dl.ary-ns-11","timezone":"UTC","time":"2024-02-02T13:13:04Z"} {"log":{"message":"Reloaded logging configuration due to SIGHUP."},"extension":{"type":"log","tags":["info","cli","config"],"pid":116},"type":"log","level":"info","system":"aryan","systemid":"3363716ae1c447e5b0ca626c6a0bf536","host":"test-11-aryan-dashboards-7797654784-6c6dl.ary-ns-11","timezone":"UTC","time":"2024-02-02T13:13:04Z"}
And after that no logs get pushed to the dashboards.log file but goes to the dashboards.log.0 file and the logs are not displayed inside the dashboards pod.
Configuration:
LOGGING_ROTATE_EVERYBYTES=1048577
LOGGING_DEST=/tmp/dboLogs/dashboards.log
LOGGING_ROTATE_ENABLED=true
Following env’s are set for opensearch dashboards.
Relevant Logs or Screenshots:
{"log":{"message":"detected rotation of /tmp/mainContainerLogs/dashboards.log; waiting 5 seconds"},"type":"log","level":"unavailable","timezone":"UTC","system":"aryan","systemid":"3363716ae1c447e5b0ca626c6a0bf536","host":"test-11-aryan-dashboards-7797654784-6c6dl.ary-ns-11","time":"2024-02-02T13:13:04+0000"} {"log":{"message":"Reloading logging configuration due to SIGHUP."},"extension":{"type":"log","tags":["info","cli","config"],"pid":116},"type":"log","level":"info","system":"aryan","systemid":"3363716ae1c447e5b0ca626c6a0bf536","host":"test-11-aryan-dashboards-7797654784-6c6dl.ary-ns-11","timezone":"UTC","time":"2024-02-02T13:13:04Z"} {"log":{"message":"following tail of /tmp/mainContainerLogs/dashboards.log"},"type":"log","level":"unavailable","timezone":"UTC","system":"aryan","systemid":"3363716ae1c447e5b0ca626c6a0bf536","host":"test-11-aryan-dashboards-7797654784-6c6dl.ary-ns-11","time":"2024-02-02T13:13:04+0000"} {"time":"2024-02-02T13:13:04+0000","level":"info","message":"following tail of /tmp/mainContainerLogs/dashboards.log","worker_id":0} {"log":{"message":"TypeError: Cannot read property 'reconfigure' of undefined\n at OsdServer.applyLoggingConfiguration (/usr/share/opensearch-dashboards/src/legacy/server/osd_server.js:182:42)\n at TapSubscriber._tapNext (/usr/share/opensearch-dashboards/src/core/server/legacy/legacy_service.js:77:24)\n at TapSubscriber._next (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/operators/tap.js:59:27)\n at TapSubscriber.Subscriber.next (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/Subscriber.js:66:18)\n at CombineLatestSubscriber.notifyNext (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/observable/combineLatest.js:97:34)\n at InnerSubscriber._next (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/InnerSubscriber.js:28:21)\n at InnerSubscriber.Subscriber.next (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/Subscriber.js:66:18)\n at ReplaySubject.Subject.next (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/Subject.js:60:25)\n at ReplaySubject.nextInfiniteTimeWindow (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/ReplaySubject.js:50:31)\n at Object.next (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/operators/shareReplay.js:36:29)\n at SafeSubscriber.__tryOrUnsub (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/Subscriber.js:205:16)\n at SafeSubscriber.next (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/Subscriber.js:143:22)\n at Subscriber._next (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/Subscriber.js:89:26)\n at Subscriber.next (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/Subscriber.js:66:18)\n at MapSubscriber._next (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/operators/map.js:55:26)\n at MapSubscriber.Subscriber.next (/usr/share/opensearch-dashboards/node_modules/rxjs/internal/Subscriber.js:66:18)"},"extension":{"type":"log","tags":["error","legacy-service"],"pid":116},"type":"log","level":"info","system":"aryan","systemid":"3363716ae1c447e5b0ca626c6a0bf536","host":"test-11-aryan-dashboards-7797654784-6c6dl.ary-ns-11","timezone":"UTC","time":"2024-02-02T13:13:04Z"} {"log":{"message":"New logging configuration:\n{\n \"ops\": {\n \"cGroupOverrides\": {\n \"cpuPath\": \"/\",\n \"cpuAcctPath\": \"/\"\n },\n \"interval\": 5000\n },\n \"logging\": {\n \"dest\": \"/tmp/mainContainerLogs/dashboards.log\",\n \"rotate\": {\n \"enabled\": true,\n \"everyBytes\": 1048577,\n \"keepFiles\": 7,\n \"pollingInterval\": 10000,\n \"usePolling\": false\n },\n \"silent\": false,\n \"quiet\": false,\n \"verbose\": false,\n \"events\": {},\n \"filter\": {},\n \"json\": true,\n \"timezone\": \"UTC\"\n }\n}"},"extension":{"type":"log","tags":["info","config"],"pid":116},"type":"log","level":"info","system":"aryan","systemid":"3363716ae1c447e5b0ca626c6a0bf536","host":"test-11-aryan-dashboards-7797654784-6c6dl.ary-ns-11","timezone":"UTC","time":"2024-02-02T13:13:04Z"} {"log":{"message":"Reloaded logging configuration due to SIGHUP."},"extension":{"type":"log","tags":["info","cli","config"],"pid":116},"type":"log","level":"info","system":"aryan","systemid":"3363716ae1c447e5b0ca626c6a0bf536","host":"test-11-aryan-dashboards-7797654784-6c6dl.ary-ns-11","timezone":"UTC","time":"2024-02-02T13:13:04Z"}