Slow query performance issue

Hi, I am moving from Elasitcsearch 7.13.4 to Opensearch 1.3.2. However, my Opensearch cluster is experiencing a slow query performance issue compared to the Elasticsearch cluster running in the same Kubernetes cluster. I have been sending the same syslog data to both Opensearch and Elasticsearch cluster for some weeks. It takes longer for Opensearch to return a result(Opensearch takes 26s, and Elasticsearch takes 0.3s). Do you have any suggestions for performance tunning? Thank you

[root@k3s-1 ~]# time curl -u $username:$password -sk https://10.43.80.73:9200/*syslog*/_search -H 'Content-Type: application/json' -d '{
  "size": 1000,
  "sort": [
    {
      "timestamp": {
        "order": "desc",
        "unmapped_type": "boolean"
      }
    }
  ],
  "stored_fields": [
    "*"
  ],
  "docvalue_fields": [
    {
      "field": "@timestamp",
      "format": "date_time"
    },
    {
      "field": "timestamp",
      "format": "date_time"
    }
  ],
  "query": {
    "bool": {
      "filter": [
        {
          "match_all": {}
        },
        {
          "range": {
            "timestamp": {
              "gte": "2022-05-22T20:53:54.323Z",
              "lte": "2022-06-05T20:53:54.323Z",
              "format": "strict_date_optional_time"
            }
          }
        }
      ]
    }
  }
}' | jq . >/tmp/es-syslog

real	0m0.368s
user	0m0.047s
sys	0m0.012s

[root@k3s-1 ~]# time curl -u $username:$password -sk https://10.43.218.222:9200/*syslog*/_search -H 'Content-Type: application/json' -d '{
  "size": 1000,
  "sort": [
    {
      "timestamp": {
        "order": "desc",
        "unmapped_type": "boolean"
      }
    }
  ],
  "stored_fields": [
    "*"
  ],
  "docvalue_fields": [
    {
      "field": "@timestamp",
      "format": "date_time"
    },
    {
      "field": "timestamp",
      "format": "date_time"
    }
  ],
  "query": {
    "bool": {
      "filter": [
        {
          "match_all": {}
        },
        {
          "range": {
            "timestamp": {
              "gte": "2022-05-22T20:53:54.323Z",
              "lte": "2022-06-05T20:53:54.323Z",
              "format": "strict_date_optional_time"
            }
          }
        }
      ]
    }
  }
}' | jq . >/tmp/os-syslog

real	0m26.110s
user	0m0.048s
sys	0m0.010s

[root@k3s-1 ~]# du -sh /tmp/*s-syslog
304K	/tmp/es-syslog
304K	/tmp/os-syslog
[root@k3s-1 ~]# kubectl get svc -A |grep 10.43.80.73
elasticsearch     elasticsearch-http                               ClusterIP      10.43.80.73     <none>                                                    9200/TCP                                                                  217d
[root@k3s-1 ~]# kubectl get svc -A |grep 10.43.218.222
opensearch     opensearch-master                             ClusterIP      10.43.218.222   <none>                                                    9200/TCP,9300/TCP                                                         18d
[root@k3s-1 ~]# 

My Opensearch is deployed using the opensearch-1.11.1 chart.

[root@k3s-1 ~]# helm list -n opensearch
NAME                    	NAMESPACE    	REVISION	UPDATED                                	STATUS  	CHART                      	APP VERSION
opensearch-client       	opensearch	3       	2022-06-01 22:30:26.839203084 -0700 PDT	deployed	opensearch-1.11.1          	1.3.2      
opensearch-data         	opensearch	3       	2022-06-01 22:30:33.988830534 -0700 PDT	deployed	opensearch-1.11.1          	1.3.2      
opensearch-master       	opensearch	4       	2022-06-01 22:28:06.608649694 -0700 PDT	deployed	opensearch-1.11.1          	1.3.2      
opensearch-dashboards	opensearch	4       	2022-06-05 13:53:16.781221079 -0700 PDT	deployed	dashboards-1.5.1	1.3.2      
[root@k3s-1 ~]# 

Like my Elasticsearch cluster, it has 8 data nodes, 3 master nodes and 2 clients, but bigger heap size(6G for Openseach Data nodes, 4G for Elasticsearch Data nodes)

# Opensearch JVM configuration
[root@k3s-1 ~]# curl -u $username:$password -sk https://10.43.218.222:9200/_nodes/_all/jvm | jq .
{
  "_nodes": {
    "total": 13,
    "successful": 13,
    "failed": 0
  },
  "cluster_name": "opensearch",
  "nodes": {
    "VmM7HkY2Ru-ttAM8AUBJag": {
      "name": "opensearch-client-1",
      "transport_address": "10.42.3.228:9300",
      "host": "10.42.3.228",
      "ip": "10.42.3.228",
      "version": "1.3.2",
      "build_type": "tar",
      "build_hash": "6febcf7b53ff189de767e460e905e9e5aeecc8cb",
      "roles": [
        "ingest"
      ],
      "attributes": {
        "shard_indexing_pressure_enabled": "true"
      },
      "jvm": {
        "pid": 129,
        "version": "11.0.15",
        "vm_name": "OpenJDK 64-Bit Server VM",
        "vm_version": "11.0.15+10",
        "vm_vendor": "Eclipse Adoptium",
        "bundled_jdk": true,
        "using_bundled_jdk": true,
        "start_time_in_millis": 1654147848459,
        "mem": {
          "heap_init_in_bytes": 4294967296,
          "heap_max_in_bytes": 4294967296,
          "non_heap_init_in_bytes": 7667712,
          "non_heap_max_in_bytes": 0,
          "direct_max_in_bytes": 0
        },
        "gc_collectors": [
          "G1 Young Generation",
          "G1 Old Generation"
        ],
        "memory_pools": [
          "CodeHeap 'non-nmethods'",
          "Metaspace",
          "CodeHeap 'profiled nmethods'",
          "Compressed Class Space",
          "G1 Eden Space",
          "G1 Old Gen",
          "G1 Survivor Space",
          "CodeHeap 'non-profiled nmethods'"
        ],
        "using_compressed_ordinary_object_pointers": "true",
        "input_arguments": [
          "-Xshare:auto",
          "-Dopensearch.networkaddress.cache.ttl=60",
          "-Dopensearch.networkaddress.cache.negative.ttl=10",
          "-XX:+AlwaysPreTouch",
          "-Xss1m",
          "-Djava.awt.headless=true",
          "-Dfile.encoding=UTF-8",
          "-Djna.nosys=true",
          "-XX:-OmitStackTraceInFastThrow",
          "-Dio.netty.noUnsafe=true",
          "-Dio.netty.noKeySetOptimization=true",
          "-Dio.netty.recycler.maxCapacityPerThread=0",
          "-Dio.netty.allocator.numDirectArenas=0",
          "-Dlog4j.shutdownHookEnabled=false",
          "-Dlog4j2.disable.jmx=true",
          "-Djava.locale.providers=SPI,COMPAT",
          "-Xms1g",
          "-Xmx1g",
          "-XX:+UseG1GC",
          "-XX:G1ReservePercent=25",
          "-XX:InitiatingHeapOccupancyPercent=30",
          "-Djava.io.tmpdir=/tmp/opensearch-9134609344391643476",
          "-XX:+HeapDumpOnOutOfMemoryError",
          "-XX:HeapDumpPath=data",
          "-XX:ErrorFile=logs/hs_err_pid%p.log",
          "-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m",
          "-Dclk.tck=100",
          "-Djdk.attach.allowAttachSelf=true",
          "-Djava.security.policy=/usr/share/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy",
          "--add-opens=jdk.attach/sun.tools.attach=ALL-UNNAMED",
          "-Dopensearch.cgroups.hierarchy.override=/",
          "-Xmx4G",
          "-Xms4G",
          "-Dopensearch.allow_insecure_settings=true",
          "-XX:MaxDirectMemorySize=2147483648",
          "-Dopensearch.path.home=/usr/share/opensearch",
          "-Dopensearch.path.conf=/usr/share/opensearch/config",
          "-Dopensearch.distribution.type=tar",
          "-Dopensearch.bundled_jdk=true"
        ]
      }
    },
    "i64sSckCQS63f8Vnrz9hjw": {
      "name": "opensearch-data-4",
      "transport_address": "10.42.5.238:9300",
      "host": "10.42.5.238",
      "ip": "10.42.5.238",
      "version": "1.3.2",
      "build_type": "tar",
      "build_hash": "6febcf7b53ff189de767e460e905e9e5aeecc8cb",
      "roles": [
        "data"
      ],
      "attributes": {
        "shard_indexing_pressure_enabled": "true"
      },
      "jvm": {
        "pid": 130,
        "version": "11.0.15",
        "vm_name": "OpenJDK 64-Bit Server VM",
        "vm_version": "11.0.15+10",
        "vm_vendor": "Eclipse Adoptium",
        "bundled_jdk": true,
        "using_bundled_jdk": true,
        "start_time_in_millis": 1654147884253,
        "mem": {
          "heap_init_in_bytes": 6442450944,
          "heap_max_in_bytes": 6442450944,
          "non_heap_init_in_bytes": 7667712,
          "non_heap_max_in_bytes": 0,
          "direct_max_in_bytes": 0
        },
        "gc_collectors": [
          "G1 Young Generation",
          "G1 Old Generation"
        ],
        "memory_pools": [
          "CodeHeap 'non-nmethods'",
          "Metaspace",
          "CodeHeap 'profiled nmethods'",
          "Compressed Class Space",
          "G1 Eden Space",
          "G1 Old Gen",
          "G1 Survivor Space",
          "CodeHeap 'non-profiled nmethods'"
        ],
        "using_compressed_ordinary_object_pointers": "true",
        "input_arguments": [
          "-Xshare:auto",
          "-Dopensearch.networkaddress.cache.ttl=60",
          "-Dopensearch.networkaddress.cache.negative.ttl=10",
          "-XX:+AlwaysPreTouch",
          "-Xss1m",
          "-Djava.awt.headless=true",
          "-Dfile.encoding=UTF-8",
          "-Djna.nosys=true",
          "-XX:-OmitStackTraceInFastThrow",
          "-Dio.netty.noUnsafe=true",
          "-Dio.netty.noKeySetOptimization=true",
          "-Dio.netty.recycler.maxCapacityPerThread=0",
          "-Dio.netty.allocator.numDirectArenas=0",
          "-Dlog4j.shutdownHookEnabled=false",
          "-Dlog4j2.disable.jmx=true",
          "-Djava.locale.providers=SPI,COMPAT",
          "-Xms1g",
          "-Xmx1g",
          "-XX:+UseG1GC",
          "-XX:G1ReservePercent=25",
          "-XX:InitiatingHeapOccupancyPercent=30",
          "-Djava.io.tmpdir=/tmp/opensearch-7566912590568961428",
          "-XX:+HeapDumpOnOutOfMemoryError",
          "-XX:HeapDumpPath=data",
          "-XX:ErrorFile=logs/hs_err_pid%p.log",
          "-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m",
          "-Dclk.tck=100",
          "-Djdk.attach.allowAttachSelf=true",
          "-Djava.security.policy=/usr/share/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy",
          "--add-opens=jdk.attach/sun.tools.attach=ALL-UNNAMED",
          "-Dopensearch.cgroups.hierarchy.override=/",
          "-Xmx6G",
          "-Xms6G",
          "-Dopensearch.allow_insecure_settings=true",
          "-XX:MaxDirectMemorySize=3221225472",
          "-Dopensearch.path.home=/usr/share/opensearch",
          "-Dopensearch.path.conf=/usr/share/opensearch/config",
          "-Dopensearch.distribution.type=tar",
          "-Dopensearch.bundled_jdk=true"
        ]
      }
    },

    "eVnJzH8mRjyk2j2e0uJ_aQ": {
      "name": "opensearch-data-1",
      "transport_address": "10.42.3.229:9300",
      "host": "10.42.3.229",
      "ip": "10.42.3.229",
      "version": "1.3.2",
      "build_type": "tar",
      "build_hash": "6febcf7b53ff189de767e460e905e9e5aeecc8cb",
      "roles": [
        "data"
      ],
      "attributes": {
        "shard_indexing_pressure_enabled": "true"
      },
      "jvm": {
        "pid": 125,
        "version": "11.0.15",
        "vm_name": "OpenJDK 64-Bit Server VM",
        "vm_version": "11.0.15+10",
        "vm_vendor": "Eclipse Adoptium",
        "bundled_jdk": true,
        "using_bundled_jdk": true,
        "start_time_in_millis": 1654147930215,
        "mem": {
          "heap_init_in_bytes": 6442450944,
          "heap_max_in_bytes": 6442450944,
          "non_heap_init_in_bytes": 7667712,
          "non_heap_max_in_bytes": 0,
          "direct_max_in_bytes": 0
        },
        "gc_collectors": [
          "G1 Young Generation",
          "G1 Old Generation"
        ],
        "memory_pools": [
          "CodeHeap 'non-nmethods'",
          "Metaspace",
          "CodeHeap 'profiled nmethods'",
          "Compressed Class Space",
          "G1 Eden Space",
          "G1 Old Gen",
          "G1 Survivor Space",
          "CodeHeap 'non-profiled nmethods'"
        ],
        "using_compressed_ordinary_object_pointers": "true",
        "input_arguments": [
          "-Xshare:auto",
          "-Dopensearch.networkaddress.cache.ttl=60",
          "-Dopensearch.networkaddress.cache.negative.ttl=10",
          "-XX:+AlwaysPreTouch",
          "-Xss1m",
          "-Djava.awt.headless=true",
          "-Dfile.encoding=UTF-8",
          "-Djna.nosys=true",
          "-XX:-OmitStackTraceInFastThrow",
          "-Dio.netty.noUnsafe=true",
          "-Dio.netty.noKeySetOptimization=true",
          "-Dio.netty.recycler.maxCapacityPerThread=0",
          "-Dio.netty.allocator.numDirectArenas=0",
          "-Dlog4j.shutdownHookEnabled=false",
          "-Dlog4j2.disable.jmx=true",
          "-Djava.locale.providers=SPI,COMPAT",
          "-Xms1g",
          "-Xmx1g",
          "-XX:+UseG1GC",
          "-XX:G1ReservePercent=25",
          "-XX:InitiatingHeapOccupancyPercent=30",
          "-Djava.io.tmpdir=/tmp/opensearch-4160313430460082222",
          "-XX:+HeapDumpOnOutOfMemoryError",
          "-XX:HeapDumpPath=data",
          "-XX:ErrorFile=logs/hs_err_pid%p.log",
          "-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m",
          "-Dclk.tck=100",
          "-Djdk.attach.allowAttachSelf=true",
          "-Djava.security.policy=/usr/share/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy",
          "--add-opens=jdk.attach/sun.tools.attach=ALL-UNNAMED",
          "-Dopensearch.cgroups.hierarchy.override=/",
          "-Xmx6G",
          "-Xms6G",
          "-Dopensearch.allow_insecure_settings=true",
          "-XX:MaxDirectMemorySize=3221225472",
          "-Dopensearch.path.home=/usr/share/opensearch",
          "-Dopensearch.path.conf=/usr/share/opensearch/config",
          "-Dopensearch.distribution.type=tar",
          "-Dopensearch.bundled_jdk=true"
        ]
      }
    },
    "gP5stZkzRZKpzFnQwC0UDg": {
      "name": "opensearch-data-0",
      "transport_address": "10.42.10.90:9300",
      "host": "10.42.10.90",
      "ip": "10.42.10.90",
      "version": "1.3.2",
      "build_type": "tar",
      "build_hash": "6febcf7b53ff189de767e460e905e9e5aeecc8cb",
      "roles": [
        "data"
      ],
      "attributes": {
        "shard_indexing_pressure_enabled": "true"
      },
      "jvm": {
        "pid": 128,
        "version": "11.0.15",
        "vm_name": "OpenJDK 64-Bit Server VM",
        "vm_version": "11.0.15+10",
        "vm_vendor": "Eclipse Adoptium",
        "bundled_jdk": true,
        "using_bundled_jdk": true,
        "start_time_in_millis": 1654147940600,
        "mem": {
          "heap_init_in_bytes": 6442450944,
          "heap_max_in_bytes": 6442450944,
          "non_heap_init_in_bytes": 7667712,
          "non_heap_max_in_bytes": 0,
          "direct_max_in_bytes": 0
        },
        "gc_collectors": [
          "G1 Young Generation",
          "G1 Old Generation"
        ],
        "memory_pools": [
          "CodeHeap 'non-nmethods'",
          "Metaspace",
          "CodeHeap 'profiled nmethods'",
          "Compressed Class Space",
          "G1 Eden Space",
          "G1 Old Gen",
          "G1 Survivor Space",
          "CodeHeap 'non-profiled nmethods'"
        ],
        "using_compressed_ordinary_object_pointers": "true",
        "input_arguments": [
          "-Xshare:auto",
          "-Dopensearch.networkaddress.cache.ttl=60",
          "-Dopensearch.networkaddress.cache.negative.ttl=10",
          "-XX:+AlwaysPreTouch",
          "-Xss1m",
          "-Djava.awt.headless=true",
          "-Dfile.encoding=UTF-8",
          "-Djna.nosys=true",
          "-XX:-OmitStackTraceInFastThrow",
          "-Dio.netty.noUnsafe=true",
          "-Dio.netty.noKeySetOptimization=true",
          "-Dio.netty.recycler.maxCapacityPerThread=0",
          "-Dio.netty.allocator.numDirectArenas=0",
          "-Dlog4j.shutdownHookEnabled=false",
          "-Dlog4j2.disable.jmx=true",
          "-Djava.locale.providers=SPI,COMPAT",
          "-Xms1g",
          "-Xmx1g",
          "-XX:+UseG1GC",
          "-XX:G1ReservePercent=25",
          "-XX:InitiatingHeapOccupancyPercent=30",
          "-Djava.io.tmpdir=/tmp/opensearch-2401278837139651339",
          "-XX:+HeapDumpOnOutOfMemoryError",
          "-XX:HeapDumpPath=data",
          "-XX:ErrorFile=logs/hs_err_pid%p.log",
          "-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m",
          "-Dclk.tck=100",
          "-Djdk.attach.allowAttachSelf=true",
          "-Djava.security.policy=/usr/share/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy",
          "--add-opens=jdk.attach/sun.tools.attach=ALL-UNNAMED",
          "-Dopensearch.cgroups.hierarchy.override=/",
          "-Xmx6G",
          "-Xms6G",
          "-Dopensearch.allow_insecure_settings=true",
          "-XX:MaxDirectMemorySize=3221225472",
          "-Dopensearch.path.home=/usr/share/opensearch",
          "-Dopensearch.path.conf=/usr/share/opensearch/config",
          "-Dopensearch.distribution.type=tar",
          "-Dopensearch.bundled_jdk=true"
        ]
      }
    },
    "eFRzGxPbT6i-MFx4pcfQZg": {
      "name": "opensearch-master-0",
      "transport_address": "10.42.10.88:9300",
      "host": "10.42.10.88",
      "ip": "10.42.10.88",
      "version": "1.3.2",
      "build_type": "tar",
      "build_hash": "6febcf7b53ff189de767e460e905e9e5aeecc8cb",
      "roles": [
        "master"
      ],
      "attributes": {
        "shard_indexing_pressure_enabled": "true"
      },
      "jvm": {
        "pid": 127,
        "version": "11.0.15",
        "vm_name": "OpenJDK 64-Bit Server VM",
        "vm_version": "11.0.15+10",
        "vm_vendor": "Eclipse Adoptium",
        "bundled_jdk": true,
        "using_bundled_jdk": true,
        "start_time_in_millis": 1654147737041,
        "mem": {
          "heap_init_in_bytes": 4294967296,
          "heap_max_in_bytes": 4294967296,
          "non_heap_init_in_bytes": 7667712,
          "non_heap_max_in_bytes": 0,
          "direct_max_in_bytes": 0
        },
        "gc_collectors": [
          "G1 Young Generation",
          "G1 Old Generation"
        ],
        "memory_pools": [
          "CodeHeap 'non-nmethods'",
          "Metaspace",
          "CodeHeap 'profiled nmethods'",
          "Compressed Class Space",
          "G1 Eden Space",
          "G1 Old Gen",
          "G1 Survivor Space",
          "CodeHeap 'non-profiled nmethods'"
        ],
        "using_compressed_ordinary_object_pointers": "true",
        "input_arguments": [
          "-Xshare:auto",
          "-Dopensearch.networkaddress.cache.ttl=60",
          "-Dopensearch.networkaddress.cache.negative.ttl=10",
          "-XX:+AlwaysPreTouch",
          "-Xss1m",
          "-Djava.awt.headless=true",
          "-Dfile.encoding=UTF-8",
          "-Djna.nosys=true",
          "-XX:-OmitStackTraceInFastThrow",
          "-Dio.netty.noUnsafe=true",
          "-Dio.netty.noKeySetOptimization=true",
          "-Dio.netty.recycler.maxCapacityPerThread=0",
          "-Dio.netty.allocator.numDirectArenas=0",
          "-Dlog4j.shutdownHookEnabled=false",
          "-Dlog4j2.disable.jmx=true",
          "-Djava.locale.providers=SPI,COMPAT",
          "-Xms1g",
          "-Xmx1g",
          "-XX:+UseG1GC",
          "-XX:G1ReservePercent=25",
          "-XX:InitiatingHeapOccupancyPercent=30",
          "-Djava.io.tmpdir=/tmp/opensearch-18143413168129157225",
          "-XX:+HeapDumpOnOutOfMemoryError",
          "-XX:HeapDumpPath=data",
          "-XX:ErrorFile=logs/hs_err_pid%p.log",
          "-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m",
          "-Dclk.tck=100",
          "-Djdk.attach.allowAttachSelf=true",
          "-Djava.security.policy=/usr/share/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy",
          "--add-opens=jdk.attach/sun.tools.attach=ALL-UNNAMED",
          "-Dopensearch.cgroups.hierarchy.override=/",
          "-Xmx4G",
          "-Xms4G",
          "-Dopensearch.allow_insecure_settings=true",
          "-XX:MaxDirectMemorySize=2147483648",
          "-Dopensearch.path.home=/usr/share/opensearch",
          "-Dopensearch.path.conf=/usr/share/opensearch/config",
          "-Dopensearch.distribution.type=tar",
          "-Dopensearch.bundled_jdk=true"
        ]
      }
    },

    "zf58fpOyRr2JblXUQtaEaQ": {
      "name": "opensearch-data-6",
      "transport_address": "10.42.7.162:9300",
      "host": "10.42.7.162",
      "ip": "10.42.7.162",
      "version": "1.3.2",
      "build_type": "tar",
      "build_hash": "6febcf7b53ff189de767e460e905e9e5aeecc8cb",
      "roles": [
        "data"
      ],
      "attributes": {
        "shard_indexing_pressure_enabled": "true"
      },
      "jvm": {
        "pid": 132,
        "version": "11.0.15",
        "vm_name": "OpenJDK 64-Bit Server VM",
        "vm_version": "11.0.15+10",
        "vm_vendor": "Eclipse Adoptium",
        "bundled_jdk": true,
        "using_bundled_jdk": true,
        "start_time_in_millis": 1654147861215,
        "mem": {
          "heap_init_in_bytes": 6442450944,
          "heap_max_in_bytes": 6442450944,
          "non_heap_init_in_bytes": 7667712,
          "non_heap_max_in_bytes": 0,
          "direct_max_in_bytes": 0
        },
        "gc_collectors": [
          "G1 Young Generation",
          "G1 Old Generation"
        ],
        "memory_pools": [
          "CodeHeap 'non-nmethods'",
          "Metaspace",
          "CodeHeap 'profiled nmethods'",
          "Compressed Class Space",
          "G1 Eden Space",
          "G1 Old Gen",
          "G1 Survivor Space",
          "CodeHeap 'non-profiled nmethods'"
        ],
        "using_compressed_ordinary_object_pointers": "true",
        "input_arguments": [
          "-Xshare:auto",
          "-Dopensearch.networkaddress.cache.ttl=60",
          "-Dopensearch.networkaddress.cache.negative.ttl=10",
          "-XX:+AlwaysPreTouch",
          "-Xss1m",
          "-Djava.awt.headless=true",
          "-Dfile.encoding=UTF-8",
          "-Djna.nosys=true",
          "-XX:-OmitStackTraceInFastThrow",
          "-Dio.netty.noUnsafe=true",
          "-Dio.netty.noKeySetOptimization=true",
          "-Dio.netty.recycler.maxCapacityPerThread=0",
          "-Dio.netty.allocator.numDirectArenas=0",
          "-Dlog4j.shutdownHookEnabled=false",
          "-Dlog4j2.disable.jmx=true",
          "-Djava.locale.providers=SPI,COMPAT",
          "-Xms1g",
          "-Xmx1g",
          "-XX:+UseG1GC",
          "-XX:G1ReservePercent=25",
          "-XX:InitiatingHeapOccupancyPercent=30",
          "-Djava.io.tmpdir=/tmp/opensearch-4945139372698676648",
          "-XX:+HeapDumpOnOutOfMemoryError",
          "-XX:HeapDumpPath=data",
          "-XX:ErrorFile=logs/hs_err_pid%p.log",
          "-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m",
          "-Dclk.tck=100",
          "-Djdk.attach.allowAttachSelf=true",
          "-Djava.security.policy=/usr/share/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy",
          "--add-opens=jdk.attach/sun.tools.attach=ALL-UNNAMED",
          "-Dopensearch.cgroups.hierarchy.override=/",
          "-Xmx6G",
          "-Xms6G",
          "-Dopensearch.allow_insecure_settings=true",
          "-XX:MaxDirectMemorySize=3221225472",
          "-Dopensearch.path.home=/usr/share/opensearch",
          "-Dopensearch.path.conf=/usr/share/opensearch/config",
          "-Dopensearch.distribution.type=tar",
          "-Dopensearch.bundled_jdk=true"
        ]
      }
    },
    "KdOcsgdSTBCPhYTZPLw4Vg": {
      "name": "opensearch-client-0",
      "transport_address": "10.42.10.89:9300",
      "host": "10.42.10.89",
      "ip": "10.42.10.89",
      "version": "1.3.2",
      "build_type": "tar",
      "build_hash": "6febcf7b53ff189de767e460e905e9e5aeecc8cb",
      "roles": [
        "ingest"
      ],
      "attributes": {
        "shard_indexing_pressure_enabled": "true"
      },
      "jvm": {
        "pid": 135,
        "version": "11.0.15",
        "vm_name": "OpenJDK 64-Bit Server VM",
        "vm_version": "11.0.15+10",
        "vm_vendor": "Eclipse Adoptium",
        "bundled_jdk": true,
        "using_bundled_jdk": true,
        "start_time_in_millis": 1654147860058,
        "mem": {
          "heap_init_in_bytes": 4294967296,
          "heap_max_in_bytes": 4294967296,
          "non_heap_init_in_bytes": 7667712,
          "non_heap_max_in_bytes": 0,
          "direct_max_in_bytes": 0
        },
        "gc_collectors": [
          "G1 Young Generation",
          "G1 Old Generation"
        ],
        "memory_pools": [
          "CodeHeap 'non-nmethods'",
          "Metaspace",
          "CodeHeap 'profiled nmethods'",
          "Compressed Class Space",
          "G1 Eden Space",
          "G1 Old Gen",
          "G1 Survivor Space",
          "CodeHeap 'non-profiled nmethods'"
        ],
        "using_compressed_ordinary_object_pointers": "true",
        "input_arguments": [
          "-Xshare:auto",
          "-Dopensearch.networkaddress.cache.ttl=60",
          "-Dopensearch.networkaddress.cache.negative.ttl=10",
          "-XX:+AlwaysPreTouch",
          "-Xss1m",
          "-Djava.awt.headless=true",
          "-Dfile.encoding=UTF-8",
          "-Djna.nosys=true",
          "-XX:-OmitStackTraceInFastThrow",
          "-Dio.netty.noUnsafe=true",
          "-Dio.netty.noKeySetOptimization=true",
          "-Dio.netty.recycler.maxCapacityPerThread=0",
          "-Dio.netty.allocator.numDirectArenas=0",
          "-Dlog4j.shutdownHookEnabled=false",
          "-Dlog4j2.disable.jmx=true",
          "-Djava.locale.providers=SPI,COMPAT",
          "-Xms1g",
          "-Xmx1g",
          "-XX:+UseG1GC",
          "-XX:G1ReservePercent=25",
          "-XX:InitiatingHeapOccupancyPercent=30",
          "-Djava.io.tmpdir=/tmp/opensearch-7094661206726421807",
          "-XX:+HeapDumpOnOutOfMemoryError",
          "-XX:HeapDumpPath=data",
          "-XX:ErrorFile=logs/hs_err_pid%p.log",
          "-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m",
          "-Dclk.tck=100",
          "-Djdk.attach.allowAttachSelf=true",
          "-Djava.security.policy=/usr/share/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy",
          "--add-opens=jdk.attach/sun.tools.attach=ALL-UNNAMED",
          "-Dopensearch.cgroups.hierarchy.override=/",
          "-Xmx4G",
          "-Xms4G",
          "-Dopensearch.allow_insecure_settings=true",
          "-XX:MaxDirectMemorySize=2147483648",
          "-Dopensearch.path.home=/usr/share/opensearch",
          "-Dopensearch.path.conf=/usr/share/opensearch/config",
          "-Dopensearch.distribution.type=tar",
          "-Dopensearch.bundled_jdk=true"
        ]
      }
    },
    "yWnsvjRFTXWOzUCDJvdMIw": {
      "name": "opensearch-master-2",
      "transport_address": "10.42.8.229:9300",
      "host": "10.42.8.229",
      "ip": "10.42.8.229",
      "version": "1.3.2",
      "build_type": "tar",
      "build_hash": "6febcf7b53ff189de767e460e905e9e5aeecc8cb",
      "roles": [
        "master"
      ],
      "attributes": {
        "shard_indexing_pressure_enabled": "true"
      },
      "jvm": {
        "pid": 131,
        "version": "11.0.15",
        "vm_name": "OpenJDK 64-Bit Server VM",
        "vm_version": "11.0.15+10",
        "vm_vendor": "Eclipse Adoptium",
        "bundled_jdk": true,
        "using_bundled_jdk": true,
        "start_time_in_millis": 1654147705365,
        "mem": {
          "heap_init_in_bytes": 4294967296,
          "heap_max_in_bytes": 4294967296,
          "non_heap_init_in_bytes": 7667712,
          "non_heap_max_in_bytes": 0,
          "direct_max_in_bytes": 0
        },
        "gc_collectors": [
          "G1 Young Generation",
          "G1 Old Generation"
        ],
        "memory_pools": [
          "CodeHeap 'non-nmethods'",
          "Metaspace",
          "CodeHeap 'profiled nmethods'",
          "Compressed Class Space",
          "G1 Eden Space",
          "G1 Old Gen",
          "G1 Survivor Space",
          "CodeHeap 'non-profiled nmethods'"
        ],
        "using_compressed_ordinary_object_pointers": "true",
        "input_arguments": [
          "-Xshare:auto",
          "-Dopensearch.networkaddress.cache.ttl=60",
          "-Dopensearch.networkaddress.cache.negative.ttl=10",
          "-XX:+AlwaysPreTouch",
          "-Xss1m",
          "-Djava.awt.headless=true",
          "-Dfile.encoding=UTF-8",
          "-Djna.nosys=true",
          "-XX:-OmitStackTraceInFastThrow",
          "-Dio.netty.noUnsafe=true",
          "-Dio.netty.noKeySetOptimization=true",
          "-Dio.netty.recycler.maxCapacityPerThread=0",
          "-Dio.netty.allocator.numDirectArenas=0",
          "-Dlog4j.shutdownHookEnabled=false",
          "-Dlog4j2.disable.jmx=true",
          "-Djava.locale.providers=SPI,COMPAT",
          "-Xms1g",
          "-Xmx1g",
          "-XX:+UseG1GC",
          "-XX:G1ReservePercent=25",
          "-XX:InitiatingHeapOccupancyPercent=30",
          "-Djava.io.tmpdir=/tmp/opensearch-14434264116468835799",
          "-XX:+HeapDumpOnOutOfMemoryError",
          "-XX:HeapDumpPath=data",
          "-XX:ErrorFile=logs/hs_err_pid%p.log",
          "-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m",
          "-Dclk.tck=100",
          "-Djdk.attach.allowAttachSelf=true",
          "-Djava.security.policy=/usr/share/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy",
          "--add-opens=jdk.attach/sun.tools.attach=ALL-UNNAMED",
          "-Dopensearch.cgroups.hierarchy.override=/",
          "-Xmx4G",
          "-Xms4G",
          "-Dopensearch.allow_insecure_settings=true",
          "-XX:MaxDirectMemorySize=2147483648",
          "-Dopensearch.path.home=/usr/share/opensearch",
          "-Dopensearch.path.conf=/usr/share/opensearch/config",
          "-Dopensearch.distribution.type=tar",
          "-Dopensearch.bundled_jdk=true"
        ]
      }
    },
    "XnMQVF8jRXqRl4O_r9gRUw": {
      "name": "opensearch-data-7",
      "transport_address": "10.42.9.167:9300",
      "host": "10.42.9.167",
      "ip": "10.42.9.167",
      "version": "1.3.2",
      "build_type": "tar",
      "build_hash": "6febcf7b53ff189de767e460e905e9e5aeecc8cb",
      "roles": [
        "data"
      ],
      "attributes": {
        "shard_indexing_pressure_enabled": "true"
      },
      "jvm": {
        "pid": 129,
        "version": "11.0.15",
        "vm_name": "OpenJDK 64-Bit Server VM",
        "vm_version": "11.0.15+10",
        "vm_vendor": "Eclipse Adoptium",
        "bundled_jdk": true,
        "using_bundled_jdk": true,
        "start_time_in_millis": 1654147853945,
        "mem": {
          "heap_init_in_bytes": 6442450944,
          "heap_max_in_bytes": 6442450944,
          "non_heap_init_in_bytes": 7667712,
          "non_heap_max_in_bytes": 0,
          "direct_max_in_bytes": 0
        },
        "gc_collectors": [
          "G1 Young Generation",
          "G1 Old Generation"
        ],
        "memory_pools": [
          "CodeHeap 'non-nmethods'",
          "Metaspace",
          "CodeHeap 'profiled nmethods'",
          "Compressed Class Space",
          "G1 Eden Space",
          "G1 Old Gen",
          "G1 Survivor Space",
          "CodeHeap 'non-profiled nmethods'"
        ],
        "using_compressed_ordinary_object_pointers": "true",
        "input_arguments": [
          "-Xshare:auto",
          "-Dopensearch.networkaddress.cache.ttl=60",
          "-Dopensearch.networkaddress.cache.negative.ttl=10",
          "-XX:+AlwaysPreTouch",
          "-Xss1m",
          "-Djava.awt.headless=true",
          "-Dfile.encoding=UTF-8",
          "-Djna.nosys=true",
          "-XX:-OmitStackTraceInFastThrow",
          "-Dio.netty.noUnsafe=true",
          "-Dio.netty.noKeySetOptimization=true",
          "-Dio.netty.recycler.maxCapacityPerThread=0",
          "-Dio.netty.allocator.numDirectArenas=0",
          "-Dlog4j.shutdownHookEnabled=false",
          "-Dlog4j2.disable.jmx=true",
          "-Djava.locale.providers=SPI,COMPAT",
          "-Xms1g",
          "-Xmx1g",
          "-XX:+UseG1GC",
          "-XX:G1ReservePercent=25",
          "-XX:InitiatingHeapOccupancyPercent=30",
          "-Djava.io.tmpdir=/tmp/opensearch-7040909685000132879",
          "-XX:+HeapDumpOnOutOfMemoryError",
          "-XX:HeapDumpPath=data",
          "-XX:ErrorFile=logs/hs_err_pid%p.log",
          "-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m",
          "-Dclk.tck=100",
          "-Djdk.attach.allowAttachSelf=true",
          "-Djava.security.policy=/usr/share/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy",
          "--add-opens=jdk.attach/sun.tools.attach=ALL-UNNAMED",
          "-Dopensearch.cgroups.hierarchy.override=/",
          "-Xmx6G",
          "-Xms6G",
          "-Dopensearch.allow_insecure_settings=true",
          "-XX:MaxDirectMemorySize=3221225472",
          "-Dopensearch.path.home=/usr/share/opensearch",
          "-Dopensearch.path.conf=/usr/share/opensearch/config",
          "-Dopensearch.distribution.type=tar",
          "-Dopensearch.bundled_jdk=true"
        ]
      }
    },
    "GGuU-uEhTP6_nek4O6iCWg": {
      "name": "opensearch-data-5",
      "transport_address": "10.42.6.86:9300",
      "host": "10.42.6.86",
      "ip": "10.42.6.86",
      "version": "1.3.2",
      "build_type": "tar",
      "build_hash": "6febcf7b53ff189de767e460e905e9e5aeecc8cb",
      "roles": [
        "data"
      ],
      "attributes": {
        "shard_indexing_pressure_enabled": "true"
      },
      "jvm": {
        "pid": 130,
        "version": "11.0.15",
        "vm_name": "OpenJDK 64-Bit Server VM",
        "vm_version": "11.0.15+10",
        "vm_vendor": "Eclipse Adoptium",
        "bundled_jdk": true,
        "using_bundled_jdk": true,
        "start_time_in_millis": 1654158223907,
        "mem": {
          "heap_init_in_bytes": 6442450944,
          "heap_max_in_bytes": 6442450944,
          "non_heap_init_in_bytes": 7667712,
          "non_heap_max_in_bytes": 0,
          "direct_max_in_bytes": 0
        },
        "gc_collectors": [
          "G1 Young Generation",
          "G1 Old Generation"
        ],
        "memory_pools": [
          "CodeHeap 'non-nmethods'",
          "Metaspace",
          "CodeHeap 'profiled nmethods'",
          "Compressed Class Space",
          "G1 Eden Space",
          "G1 Old Gen",
          "G1 Survivor Space",
          "CodeHeap 'non-profiled nmethods'"
        ],
        "using_compressed_ordinary_object_pointers": "true",
        "input_arguments": [
          "-Xshare:auto",
          "-Dopensearch.networkaddress.cache.ttl=60",
          "-Dopensearch.networkaddress.cache.negative.ttl=10",
          "-XX:+AlwaysPreTouch",
          "-Xss1m",
          "-Djava.awt.headless=true",
          "-Dfile.encoding=UTF-8",
          "-Djna.nosys=true",
          "-XX:-OmitStackTraceInFastThrow",
          "-Dio.netty.noUnsafe=true",
          "-Dio.netty.noKeySetOptimization=true",
          "-Dio.netty.recycler.maxCapacityPerThread=0",
          "-Dio.netty.allocator.numDirectArenas=0",
          "-Dlog4j.shutdownHookEnabled=false",
          "-Dlog4j2.disable.jmx=true",
          "-Djava.locale.providers=SPI,COMPAT",
          "-Xms1g",
          "-Xmx1g",
          "-XX:+UseG1GC",
          "-XX:G1ReservePercent=25",
          "-XX:InitiatingHeapOccupancyPercent=30",
          "-Djava.io.tmpdir=/tmp/opensearch-12251988097308684762",
          "-XX:+HeapDumpOnOutOfMemoryError",
          "-XX:HeapDumpPath=data",
          "-XX:ErrorFile=logs/hs_err_pid%p.log",
          "-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m",
          "-Dclk.tck=100",
          "-Djdk.attach.allowAttachSelf=true",
          "-Djava.security.policy=/usr/share/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy",
          "--add-opens=jdk.attach/sun.tools.attach=ALL-UNNAMED",
          "-Dopensearch.cgroups.hierarchy.override=/",
          "-Xmx6G",
          "-Xms6G",
          "-Dopensearch.allow_insecure_settings=true",
          "-XX:MaxDirectMemorySize=3221225472",
          "-Dopensearch.path.home=/usr/share/opensearch",
          "-Dopensearch.path.conf=/usr/share/opensearch/config",
          "-Dopensearch.distribution.type=tar",
          "-Dopensearch.bundled_jdk=true"
        ]
      }
    },
    "UcYr5uEBQRqcRk75QtdGJQ": {
      "name": "opensearch-master-1",
      "transport_address": "10.42.3.227:9300",
      "host": "10.42.3.227",
      "ip": "10.42.3.227",
      "version": "1.3.2",
      "build_type": "tar",
      "build_hash": "6febcf7b53ff189de767e460e905e9e5aeecc8cb",
      "roles": [
        "master"
      ],
      "attributes": {
        "shard_indexing_pressure_enabled": "true"
      },
      "jvm": {
        "pid": 127,
        "version": "11.0.15",
        "vm_name": "OpenJDK 64-Bit Server VM",
        "vm_version": "11.0.15+10",
        "vm_vendor": "Eclipse Adoptium",
        "bundled_jdk": true,
        "using_bundled_jdk": true,
        "start_time_in_millis": 1654147721778,
        "mem": {
          "heap_init_in_bytes": 4294967296,
          "heap_max_in_bytes": 4294967296,
          "non_heap_init_in_bytes": 7667712,
          "non_heap_max_in_bytes": 0,
          "direct_max_in_bytes": 0
        },
        "gc_collectors": [
          "G1 Young Generation",
          "G1 Old Generation"
        ],
        "memory_pools": [
          "CodeHeap 'non-nmethods'",
          "Metaspace",
          "CodeHeap 'profiled nmethods'",
          "Compressed Class Space",
          "G1 Eden Space",
          "G1 Old Gen",
          "G1 Survivor Space",
          "CodeHeap 'non-profiled nmethods'"
        ],
        "using_compressed_ordinary_object_pointers": "true",
        "input_arguments": [
          "-Xshare:auto",
          "-Dopensearch.networkaddress.cache.ttl=60",
          "-Dopensearch.networkaddress.cache.negative.ttl=10",
          "-XX:+AlwaysPreTouch",
          "-Xss1m",
          "-Djava.awt.headless=true",
          "-Dfile.encoding=UTF-8",
          "-Djna.nosys=true",
          "-XX:-OmitStackTraceInFastThrow",
          "-Dio.netty.noUnsafe=true",
          "-Dio.netty.noKeySetOptimization=true",
          "-Dio.netty.recycler.maxCapacityPerThread=0",
          "-Dio.netty.allocator.numDirectArenas=0",
          "-Dlog4j.shutdownHookEnabled=false",
          "-Dlog4j2.disable.jmx=true",
          "-Djava.locale.providers=SPI,COMPAT",
          "-Xms1g",
          "-Xmx1g",
          "-XX:+UseG1GC",
          "-XX:G1ReservePercent=25",
          "-XX:InitiatingHeapOccupancyPercent=30",
          "-Djava.io.tmpdir=/tmp/opensearch-1841330876957653590",
          "-XX:+HeapDumpOnOutOfMemoryError",
          "-XX:HeapDumpPath=data",
          "-XX:ErrorFile=logs/hs_err_pid%p.log",
          "-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m",
          "-Dclk.tck=100",
          "-Djdk.attach.allowAttachSelf=true",
          "-Djava.security.policy=/usr/share/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy",
          "--add-opens=jdk.attach/sun.tools.attach=ALL-UNNAMED",
          "-Dopensearch.cgroups.hierarchy.override=/",
          "-Xmx4G",
          "-Xms4G",
          "-Dopensearch.allow_insecure_settings=true",
          "-XX:MaxDirectMemorySize=2147483648",
          "-Dopensearch.path.home=/usr/share/opensearch",
          "-Dopensearch.path.conf=/usr/share/opensearch/config",
          "-Dopensearch.distribution.type=tar",
          "-Dopensearch.bundled_jdk=true"
        ]
      }
    },
    "0dw_Kx19S7WYsa3XkvLC6Q": {
      "name": "opensearch-data-3",
      "transport_address": "10.42.4.189:9300",
      "host": "10.42.4.189",
      "ip": "10.42.4.189",
      "version": "1.3.2",
      "build_type": "tar",
      "build_hash": "6febcf7b53ff189de767e460e905e9e5aeecc8cb",
      "roles": [
        "data"
      ],
      "attributes": {
        "shard_indexing_pressure_enabled": "true"
      },
      "jvm": {
        "pid": 128,
        "version": "11.0.15",
        "vm_name": "OpenJDK 64-Bit Server VM",
        "vm_version": "11.0.15+10",
        "vm_vendor": "Eclipse Adoptium",
        "bundled_jdk": true,
        "using_bundled_jdk": true,
        "start_time_in_millis": 1654147894019,
        "mem": {
          "heap_init_in_bytes": 6442450944,
          "heap_max_in_bytes": 6442450944,
          "non_heap_init_in_bytes": 7667712,
          "non_heap_max_in_bytes": 0,
          "direct_max_in_bytes": 0
        },
        "gc_collectors": [
          "G1 Young Generation",
          "G1 Old Generation"
        ],
        "memory_pools": [
          "CodeHeap 'non-nmethods'",
          "Metaspace",
          "CodeHeap 'profiled nmethods'",
          "Compressed Class Space",
          "G1 Eden Space",
          "G1 Old Gen",
          "G1 Survivor Space",
          "CodeHeap 'non-profiled nmethods'"
        ],
        "using_compressed_ordinary_object_pointers": "true",
        "input_arguments": [
          "-Xshare:auto",
          "-Dopensearch.networkaddress.cache.ttl=60",
          "-Dopensearch.networkaddress.cache.negative.ttl=10",
          "-XX:+AlwaysPreTouch",
          "-Xss1m",
          "-Djava.awt.headless=true",
          "-Dfile.encoding=UTF-8",
          "-Djna.nosys=true",
          "-XX:-OmitStackTraceInFastThrow",
          "-Dio.netty.noUnsafe=true",
          "-Dio.netty.noKeySetOptimization=true",
          "-Dio.netty.recycler.maxCapacityPerThread=0",
          "-Dio.netty.allocator.numDirectArenas=0",
          "-Dlog4j.shutdownHookEnabled=false",
          "-Dlog4j2.disable.jmx=true",
          "-Djava.locale.providers=SPI,COMPAT",
          "-Xms1g",
          "-Xmx1g",
          "-XX:+UseG1GC",
          "-XX:G1ReservePercent=25",
          "-XX:InitiatingHeapOccupancyPercent=30",
          "-Djava.io.tmpdir=/tmp/opensearch-10078783294544779684",
          "-XX:+HeapDumpOnOutOfMemoryError",
          "-XX:HeapDumpPath=data",
          "-XX:ErrorFile=logs/hs_err_pid%p.log",
          "-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m",
          "-Dclk.tck=100",
          "-Djdk.attach.allowAttachSelf=true",
          "-Djava.security.policy=/usr/share/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy",
          "--add-opens=jdk.attach/sun.tools.attach=ALL-UNNAMED",
          "-Dopensearch.cgroups.hierarchy.override=/",
          "-Xmx6G",
          "-Xms6G",
          "-Dopensearch.allow_insecure_settings=true",
          "-XX:MaxDirectMemorySize=3221225472",
          "-Dopensearch.path.home=/usr/share/opensearch",
          "-Dopensearch.path.conf=/usr/share/opensearch/config",
          "-Dopensearch.distribution.type=tar",
          "-Dopensearch.bundled_jdk=true"
        ]
      }
    },
    "ASZqaxEZQEm9eyt5DsUvKQ": {
      "name": "opensearch-data-2",
      "transport_address": "10.42.8.230:9300",
      "host": "10.42.8.230",
      "ip": "10.42.8.230",
      "version": "1.3.2",
      "build_type": "tar",
      "build_hash": "6febcf7b53ff189de767e460e905e9e5aeecc8cb",
      "roles": [
        "data"
      ],
      "attributes": {
        "shard_indexing_pressure_enabled": "true"
      },
      "jvm": {
        "pid": 125,
        "version": "11.0.15",
        "vm_name": "OpenJDK 64-Bit Server VM",
        "vm_version": "11.0.15+10",
        "vm_vendor": "Eclipse Adoptium",
        "bundled_jdk": true,
        "using_bundled_jdk": true,
        "start_time_in_millis": 1654147910738,
        "mem": {
          "heap_init_in_bytes": 6442450944,
          "heap_max_in_bytes": 6442450944,
          "non_heap_init_in_bytes": 7667712,
          "non_heap_max_in_bytes": 0,
          "direct_max_in_bytes": 0
        },
        "gc_collectors": [
          "G1 Young Generation",
          "G1 Old Generation"
        ],
        "memory_pools": [
          "CodeHeap 'non-nmethods'",
          "Metaspace",
          "CodeHeap 'profiled nmethods'",
          "Compressed Class Space",
          "G1 Eden Space",
          "G1 Old Gen",
          "G1 Survivor Space",
          "CodeHeap 'non-profiled nmethods'"
        ],
        "using_compressed_ordinary_object_pointers": "true",
        "input_arguments": [
          "-Xshare:auto",
          "-Dopensearch.networkaddress.cache.ttl=60",
          "-Dopensearch.networkaddress.cache.negative.ttl=10",
          "-XX:+AlwaysPreTouch",
          "-Xss1m",
          "-Djava.awt.headless=true",
          "-Dfile.encoding=UTF-8",
          "-Djna.nosys=true",
          "-XX:-OmitStackTraceInFastThrow",
          "-Dio.netty.noUnsafe=true",
          "-Dio.netty.noKeySetOptimization=true",
          "-Dio.netty.recycler.maxCapacityPerThread=0",
          "-Dio.netty.allocator.numDirectArenas=0",
          "-Dlog4j.shutdownHookEnabled=false",
          "-Dlog4j2.disable.jmx=true",
          "-Djava.locale.providers=SPI,COMPAT",
          "-Xms1g",
          "-Xmx1g",
          "-XX:+UseG1GC",
          "-XX:G1ReservePercent=25",
          "-XX:InitiatingHeapOccupancyPercent=30",
          "-Djava.io.tmpdir=/tmp/opensearch-3720998071567621476",
          "-XX:+HeapDumpOnOutOfMemoryError",
          "-XX:HeapDumpPath=data",
          "-XX:ErrorFile=logs/hs_err_pid%p.log",
          "-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m",
          "-Dclk.tck=100",
          "-Djdk.attach.allowAttachSelf=true",
          "-Djava.security.policy=/usr/share/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy",
          "--add-opens=jdk.attach/sun.tools.attach=ALL-UNNAMED",
          "-Dopensearch.cgroups.hierarchy.override=/",
          "-Xmx6G",
          "-Xms6G",
          "-Dopensearch.allow_insecure_settings=true",
          "-XX:MaxDirectMemorySize=3221225472",
          "-Dopensearch.path.home=/usr/share/opensearch",
          "-Dopensearch.path.conf=/usr/share/opensearch/config",
          "-Dopensearch.distribution.type=tar",
          "-Dopensearch.bundled_jdk=true"
        ]
      }
    }
  }
}

Hi @deng47 ,

This is interesting, could you please run your query with profile enabled [1] and share the breakdown? Thank you.

[1] Profile API | Elasticsearch Guide [7.10] | Elastic

@reta

Thank you for your reply. I have deleted my data in OpenSearch, so I can’t compare the performance at this moment. I will start flowing data into Opensearch for a few days, and try enabling the profile. I will put my update here

I started a new OpenSearch cluster and still see a performance issue.
I query all data in a 24-hour window. In the discover tab, it takes around 30 seconds and returns 1.6 billion hits.


I put the same request with profile enabled in the console tab, it takes almost 1 min. Below is my query

GET /my-system-syslog/_search
{
  "profile": true,
  "version": true,
  "size": 500,
  "sort": [
    {
      "timestamp": {
        "order": "desc",
        "unmapped_type": "boolean"
      }
    }
  ],
  "aggs": {
    "2": {
      "date_histogram": {
        "field": "timestamp",
        "fixed_interval": "12h",
        "time_zone": "America/Los_Angeles",
        "min_doc_count": 1
      }
    }
  },
  "stored_fields": [
    "*"
  ],
  "script_fields": {},
  "docvalue_fields": [
    {
      "field": "@timestamp",
      "format": "date_time"
    },
    {
      "field": "timestamp",
      "format": "date_time"
    }
  ],
  "_source": {
    "excludes": []
  },
  "query": {
    "bool": {
      "must": [],
      "filter": [
        {
          "match_all": {}
        },
        {
          "range": {
            "timestamp": {              
              "gte": "2023-01-20T04:00:00.000Z",
              "lte": "2023-01-21T04:00:00.000Z",
              "format": "strict_date_optional_time"
            }
          }
        }
      ],
      "should": [],
      "must_not": []
    }
  },
  "highlight": {
    "pre_tags": [
      "@opensearch-dashboards-highlighted-field@"
    ],
    "post_tags": [
      "@/opensearch-dashboards-highlighted-field@"
    ],
    "fields": {
      "*": {}
    },
    "fragment_size": 2147483647
  }
}

The profile result is too long to post here. I wonder what you are interested int

{
   "took":55608,
   "timed_out":false,
   "_shards":{
      "total":63,
      "successful":63,
      "skipped":30,
      "failed":0
   },
   "hits":{
      "total":{
         "value":10000,
         "relation":"gte"
      },
      "max_score":null,
      "hits": <removed due to len limit>
      
   },
   "aggregations":{
      "2":{
         "buckets":[
            {
               "key_as_string":"2023-01-19T12:00:00.000-08:00",
               "key":1674158400000,
               "doc_count":222573445
            },
            {
               "key_as_string":"2023-01-20T00:00:00.000-08:00",
               "key":1674201600000,
               "doc_count":986044335
            },
            {
               "key_as_string":"2023-01-20T12:00:00.000-08:00",
               "key":1674244800000,
               "doc_count":400196146
            }
         ]
      }
   },
   "profile":{
      "shards":[
         {
            "id":"[5pwR-7jcQraOj--MGQVrjw][my-system-syslog-000015][1]",
            "inbound_network_time_in_millis":0,
            "outbound_network_time_in_millis":3,
            "searches":[
               {
                  "query":[
                     {
                        "type":"BoostQuery",
                        "description":"(ConstantScore(timestamp:[1674187200000 TO 1674273600000]))^0.0",
                        "time_in_nanos":8826435998,
                        "breakdown":{
                           "set_min_competitive_score_count":0,
                           "match_count":0,
                           "shallow_advance_count":0,
                           "set_min_competitive_score":0,
                           "next_doc":8783511318,
                           "match":0,
                           "next_doc_count":112080564,
                           "score_count":0,
                           "compute_max_score_count":0,
                           "compute_max_score":0,
                           "advance":5530,
                           "advance_count":36,
                           "score":0,
                           "build_scorer_count":72,
                           "create_weight":16250,
                           "shallow_advance":0,
                           "create_weight_count":1,
                           "build_scorer":42902900
                        },
                        "children":[
                           {
                              "type":"IndexOrDocValuesQuery",
                              "description":"timestamp:[1674187200000 TO 1674273600000]",
                              "time_in_nanos":3030893489,
                              "breakdown":{
                                 "set_min_competitive_score_count":0,
                                 "match_count":0,
                                 "shallow_advance_count":0,
                                 "set_min_competitive_score":0,
                                 "next_doc":2988031159,
                                 "match":0,
                                 "next_doc_count":112080564,
                                 "score_count":0,
                                 "compute_max_score_count":0,
                                 "compute_max_score":0,
                                 "advance":2960,
                                 "advance_count":36,
                                 "score":0,
                                 "build_scorer_count":72,
                                 "create_weight":1910,
                                 "shallow_advance":0,
                                 "create_weight_count":1,
                                 "build_scorer":42857460
                              }
                           }
                        ]
                     }
                  ],
                  "rewrite_time":56269,
                  "collector":[
                     {
                        "name":"MultiCollector",
                        "reason":"search_multi",
                        "time_in_nanos":18468889102,
                        "children":[
                           {
                              "name":"SimpleFieldCollector",
                              "reason":"search_top_hits",
                              "time_in_nanos":3980105371
                           },
                           {
                              "name":"ProfilingAggregator: [2]",
                              "reason":"aggregation",
                              "time_in_nanos":5522970193
                           }
                        ]
                     }
                  ]
               }
            ],
            "aggregations":[
               {
                  "type":"DateHistogramAggregator",
                  "description":"2",
                  "time_in_nanos":5031574636,
                  "breakdown":{
                     "reduce":0,
                     "post_collection_count":1,
                     "build_leaf_collector":24800,
                     "build_aggregation":12410,
                     "build_aggregation_count":1,
                     "build_leaf_collector_count":36,
                     "post_collection":960,
                     "initialize":1630,
                     "initialize_count":1,
                     "reduce_count":0,
                     "collect":5031534836,
                     "collect_count":112080564
                  },
                  "debug":{
                     "total_buckets":2
                  }
               }
            ]
         },
         {
            "id":"[6brGb5iXQkuT9g-r-7gmLQ][my-system-syslog-000005][0]",
            "inbound_network_time_in_millis":0,
            "outbound_network_time_in_millis":2,
            "searches":[
               {
                  "query":[
                     {
                        "type":"BoostQuery",
                        "description":"(ConstantScore(timestamp:[1674187200000 TO 1674273600000]))^0.0",
                        "time_in_nanos":366440278,
                        "breakdown":{
                           "set_min_competitive_score_count":0,
                           "match_count":0,
                           "shallow_advance_count":0,
                           "set_min_competitive_score":0,
                           "next_doc":0,
                           "match":0,
                           "next_doc_count":0,
                           "score_count":0,
                           "compute_max_score_count":0,
                           "compute_max_score":0,
                           "advance":10400,
                           "advance_count":36,
                           "score":0,
                           "build_scorer_count":72,
                           "create_weight":19800,
                           "shallow_advance":0,
                           "create_weight_count":1,
                           "build_scorer":366410078
                        },
                        "children":[
                           {
                              "type":"IndexOrDocValuesQuery",
                              "description":"timestamp:[1674187200000 TO 1674273600000]",
                              "time_in_nanos":366359808,
                              "breakdown":{
                                 "set_min_competitive_score_count":0,
                                 "match_count":0,
                                 "shallow_advance_count":0,
                                 "set_min_competitive_score":0,
                                 "next_doc":0,
                                 "match":0,
                                 "next_doc_count":0,
                                 "score_count":0,
                                 "compute_max_score_count":0,
                                 "compute_max_score":0,
                                 "advance":5990,
                                 "advance_count":36,
                                 "score":0,
                                 "build_scorer_count":72,
                                 "create_weight":6000,
                                 "shallow_advance":0,
                                 "create_weight_count":1,
                                 "build_scorer":366347818
                              }
                           }
                        ]
                     }
                  ],
                  "rewrite_time":27359,
                  "collector":[
                     {
                        "name":"MultiCollector",
                        "reason":"search_multi",
                        "time_in_nanos":548423,
                        "children":[
                           {
                              "name":"SimpleFieldCollector",
                              "reason":"search_top_hits",
                              "time_in_nanos":358746
                           },
                           {
                              "name":"ProfilingAggregator: [2]",
                              "reason":"aggregation",
                              "time_in_nanos":88039
                           }
                        ]
                     }
                  ]
               }
            ],
            "aggregations":[
               {
                  "type":"DateHistogramAggregator",
                  "description":"2",
                  "time_in_nanos":108438,
                  "breakdown":{
                     "reduce":0,
                     "post_collection_count":1,
                     "build_leaf_collector":54930,
                     "build_aggregation":43219,
                     "build_aggregation_count":1,
                     "build_leaf_collector_count":36,
                     "post_collection":5030,
                     "initialize":5259,
                     "initialize_count":1,
                     "reduce_count":0,
                     "collect":0,
                     "collect_count":0
                  },
                  "debug":{
                     "total_buckets":0
                  }
               }
            ]
         },

I am running a 2.4.1 OpenSearch

GET /
{
  "name" : "my-system-os-nodes-8",
  "cluster_name" : "my-system-os",
  "cluster_uuid" : "JkTw7iprRmG86RmfP3ddzQ",
  "version" : {
    "distribution" : "opensearch",
    "number" : "2.4.1",
    "build_type" : "tar",
    "build_hash" : "f2f809ea280ffba217451da894a5899f1cec02ab",
    "build_date" : "2022-12-12T22:17:42.341124910Z",
    "build_snapshot" : false,
    "lucene_version" : "9.4.2",
    "minimum_wire_compatibility_version" : "7.10.0",
    "minimum_index_compatibility_version" : "7.0.0"
  },
  "tagline" : "The OpenSearch Project: https://opensearch.org/"
}

The cluster deployed in a Kubernetes env with opensearch-operator-2.1.1.
It has 16 data nodes, 3 masters, and 3 coordinators.
I set a 8g heap size for all data nodes, 4g for coordinators and 2g for masters with the opensearch-operator. But in the JVM output below, it shows both the default 1g heap size and my custom heap size in the input_arguments list. So I am not sure if that’s the cause of the performance issue.

GET _nodes/_all/jvm

    "jeIw6kBqQYSWloVniRxQVg" : {
      "name" : "my-system-os-coordinators-2",
      "transport_address" : "10.42.9.30:9300",
      "host" : "my-system-os-coordinators-2",
      "ip" : "10.42.9.30",
      "version" : "2.4.1",
      "build_type" : "tar",
      "build_hash" : "f2f809ea280ffba217451da894a5899f1cec02ab",
      "roles" : [
        "ingest"
      ],
      "attributes" : {
        "shard_indexing_pressure_enabled" : "true"
      },
      "jvm" : {
        "pid" : 113,
        "version" : "17.0.5",
        "vm_name" : "OpenJDK 64-Bit Server VM",
        "vm_version" : "17.0.5+8",
        "vm_vendor" : "Eclipse Adoptium",
        "bundled_jdk" : true,
        "using_bundled_jdk" : true,
        "start_time_in_millis" : 1673406754730,
        "mem" : {
          "heap_init_in_bytes" : 4294967296,
          "heap_max_in_bytes" : 4294967296,
          "non_heap_init_in_bytes" : 7667712,
          "non_heap_max_in_bytes" : 0,
          "direct_max_in_bytes" : 0
        },
        "gc_collectors" : [
          "G1 Young Generation",
          "G1 Old Generation"
        ],
        "memory_pools" : [
          "CodeHeap 'non-nmethods'",
          "Metaspace",
          "CodeHeap 'profiled nmethods'",
          "Compressed Class Space",
          "G1 Eden Space",
          "G1 Old Gen",
          "G1 Survivor Space",
          "CodeHeap 'non-profiled nmethods'"
        ],
        "using_compressed_ordinary_object_pointers" : "true",
        "input_arguments" : [
          "-Xshare:auto",
          "-Dopensearch.networkaddress.cache.ttl=60",
          "-Dopensearch.networkaddress.cache.negative.ttl=10",
          "-XX:+AlwaysPreTouch",
          "-Xss1m",
          "-Djava.awt.headless=true",
          "-Dfile.encoding=UTF-8",
          "-Djna.nosys=true",
          "-XX:-OmitStackTraceInFastThrow",
          "-XX:+ShowCodeDetailsInExceptionMessages",
          "-Dio.netty.noUnsafe=true",
          "-Dio.netty.noKeySetOptimization=true",
          "-Dio.netty.recycler.maxCapacityPerThread=0",
          "-Dio.netty.allocator.numDirectArenas=0",
          "-Dlog4j.shutdownHookEnabled=false",
          "-Dlog4j2.disable.jmx=true",
          "-Djava.locale.providers=SPI,COMPAT",
          "-Xms1g",
          "-Xmx1g",
          "-XX:+UseG1GC",
          "-XX:G1ReservePercent=25",
          "-XX:InitiatingHeapOccupancyPercent=30",
          "-Djava.io.tmpdir=/tmp/opensearch-6579018218015609015",
          "-XX:+HeapDumpOnOutOfMemoryError",
          "-XX:HeapDumpPath=data",
          "-XX:ErrorFile=logs/hs_err_pid%p.log",
          "-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m",
          "-Dclk.tck=100",
          "-Djdk.attach.allowAttachSelf=true",
          "-Djava.security.policy=/usr/share/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy",
          "--add-opens=jdk.attach/sun.tools.attach=ALL-UNNAMED",
          "-Dopensearch.cgroups.hierarchy.override=/",
          "-Xmx4g",
          "-Xms4g",
          "-Dopensearch.allow_insecure_settings=true",
          "-Dopensearch.transport.cname_in_publish_address=true",
          "-XX:MaxDirectMemorySize=2147483648",
          "-Dopensearch.path.home=/usr/share/opensearch",
          "-Dopensearch.path.conf=/usr/share/opensearch/config",
          "-Dopensearch.distribution.type=tar",
          "-Dopensearch.bundled_jdk=true"
        ]
      }
    },
    "bifQA3XVQQKuI_EMhxyGlg" : {
      "name" : "my-system-os-nodes-8",
      "transport_address" : "10.42.17.46:9300",
      "host" : "my-system-os-nodes-8",
      "ip" : "10.42.17.46",
      "version" : "2.4.1",
      "build_type" : "tar",
      "build_hash" : "f2f809ea280ffba217451da894a5899f1cec02ab",
      "roles" : [
        "data"
      ],
      "attributes" : {
        "shard_indexing_pressure_enabled" : "true"
      },
      "jvm" : {
        "pid" : 112,
        "version" : "17.0.5",
        "vm_name" : "OpenJDK 64-Bit Server VM",
        "vm_version" : "17.0.5+8",
        "vm_vendor" : "Eclipse Adoptium",
        "bundled_jdk" : true,
        "using_bundled_jdk" : true,
        "start_time_in_millis" : 1673407955837,
        "mem" : {
          "heap_init_in_bytes" : 8589934592,
          "heap_max_in_bytes" : 8589934592,
          "non_heap_init_in_bytes" : 7667712,
          "non_heap_max_in_bytes" : 0,
          "direct_max_in_bytes" : 0
        },
        "gc_collectors" : [
          "G1 Young Generation",
          "G1 Old Generation"
        ],
        "memory_pools" : [
          "CodeHeap 'non-nmethods'",
          "Metaspace",
          "CodeHeap 'profiled nmethods'",
          "Compressed Class Space",
          "G1 Eden Space",
          "G1 Old Gen",
          "G1 Survivor Space",
          "CodeHeap 'non-profiled nmethods'"
        ],
        "using_compressed_ordinary_object_pointers" : "true",
        "input_arguments" : [
          "-Xshare:auto",
          "-Dopensearch.networkaddress.cache.ttl=60",
          "-Dopensearch.networkaddress.cache.negative.ttl=10",
          "-XX:+AlwaysPreTouch",
          "-Xss1m",
          "-Djava.awt.headless=true",
          "-Dfile.encoding=UTF-8",
          "-Djna.nosys=true",
          "-XX:-OmitStackTraceInFastThrow",
          "-XX:+ShowCodeDetailsInExceptionMessages",
          "-Dio.netty.noUnsafe=true",
          "-Dio.netty.noKeySetOptimization=true",
          "-Dio.netty.recycler.maxCapacityPerThread=0",
          "-Dio.netty.allocator.numDirectArenas=0",
          "-Dlog4j.shutdownHookEnabled=false",
          "-Dlog4j2.disable.jmx=true",
          "-Djava.locale.providers=SPI,COMPAT",
          "-Xms1g",
          "-Xmx1g",
          "-XX:+UseG1GC",
          "-XX:G1ReservePercent=25",
          "-XX:InitiatingHeapOccupancyPercent=30",
          "-Djava.io.tmpdir=/tmp/opensearch-4827850128703245728",
          "-XX:+HeapDumpOnOutOfMemoryError",
          "-XX:HeapDumpPath=data",
          "-XX:ErrorFile=logs/hs_err_pid%p.log",
          "-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m",
          "-Dclk.tck=100",
          "-Djdk.attach.allowAttachSelf=true",
          "-Djava.security.policy=/usr/share/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy",
          "--add-opens=jdk.attach/sun.tools.attach=ALL-UNNAMED",
          "-Dopensearch.cgroups.hierarchy.override=/",
          "-Xmx8g",
          "-Xms8g",
          "-Dopensearch.allow_insecure_settings=true",
          "-Dopensearch.transport.cname_in_publish_address=true",
          "-XX:MaxDirectMemorySize=4294967296",
          "-Dopensearch.path.home=/usr/share/opensearch",
          "-Dopensearch.path.conf=/usr/share/opensearch/config",
          "-Dopensearch.distribution.type=tar",
          "-Dopensearch.bundled_jdk=true"
        ]
      }
    },
    "eXHDHckATeO2gq0tQSb6Yw" : {
      "name" : "my-system-os-masters-1",
      "transport_address" : "10.42.6.30:9300",
      "host" : "my-system-os-masters-1",
      "ip" : "10.42.6.30",
      "version" : "2.4.1",
      "build_type" : "tar",
      "build_hash" : "f2f809ea280ffba217451da894a5899f1cec02ab",
      "roles" : [
        "cluster_manager"
      ],
      "attributes" : {
        "shard_indexing_pressure_enabled" : "true"
      },
      "jvm" : {
        "pid" : 113,
        "version" : "17.0.5",
        "vm_name" : "OpenJDK 64-Bit Server VM",
        "vm_version" : "17.0.5+8",
        "vm_vendor" : "Eclipse Adoptium",
        "bundled_jdk" : true,
        "using_bundled_jdk" : true,
        "start_time_in_millis" : 1673406845890,
        "mem" : {
          "heap_init_in_bytes" : 2147483648,
          "heap_max_in_bytes" : 2147483648,
          "non_heap_init_in_bytes" : 7667712,
          "non_heap_max_in_bytes" : 0,
          "direct_max_in_bytes" : 0
        },
        "gc_collectors" : [
          "G1 Young Generation",
          "G1 Old Generation"
        ],
        "memory_pools" : [
          "CodeHeap 'non-nmethods'",
          "Metaspace",
          "CodeHeap 'profiled nmethods'",
          "Compressed Class Space",
          "G1 Eden Space",
          "G1 Old Gen",
          "G1 Survivor Space",
          "CodeHeap 'non-profiled nmethods'"
        ],
        "using_compressed_ordinary_object_pointers" : "true",
        "input_arguments" : [
          "-Xshare:auto",
          "-Dopensearch.networkaddress.cache.ttl=60",
          "-Dopensearch.networkaddress.cache.negative.ttl=10",
          "-XX:+AlwaysPreTouch",
          "-Xss1m",
          "-Djava.awt.headless=true",
          "-Dfile.encoding=UTF-8",
          "-Djna.nosys=true",
          "-XX:-OmitStackTraceInFastThrow",
          "-XX:+ShowCodeDetailsInExceptionMessages",
          "-Dio.netty.noUnsafe=true",
          "-Dio.netty.noKeySetOptimization=true",
          "-Dio.netty.recycler.maxCapacityPerThread=0",
          "-Dio.netty.allocator.numDirectArenas=0",
          "-Dlog4j.shutdownHookEnabled=false",
          "-Dlog4j2.disable.jmx=true",
          "-Djava.locale.providers=SPI,COMPAT",
          "-Xms1g",
          "-Xmx1g",
          "-XX:+UseG1GC",
          "-XX:G1ReservePercent=25",
          "-XX:InitiatingHeapOccupancyPercent=30",
          "-Djava.io.tmpdir=/tmp/opensearch-17425629546509655767",
          "-XX:+HeapDumpOnOutOfMemoryError",
          "-XX:HeapDumpPath=data",
          "-XX:ErrorFile=logs/hs_err_pid%p.log",
          "-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m",
          "-Dclk.tck=100",
          "-Djdk.attach.allowAttachSelf=true",
          "-Djava.security.policy=/usr/share/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy",
          "--add-opens=jdk.attach/sun.tools.attach=ALL-UNNAMED",
          "-Dopensearch.cgroups.hierarchy.override=/",
          "-Xmx2g",
          "-Xms2g",
          "-Dopensearch.allow_insecure_settings=true",
          "-Dopensearch.transport.cname_in_publish_address=true",
          "-XX:MaxDirectMemorySize=1073741824",
          "-Dopensearch.path.home=/usr/share/opensearch",
          "-Dopensearch.path.conf=/usr/share/opensearch/config",
          "-Dopensearch.distribution.type=tar",
          "-Dopensearch.bundled_jdk=true"
        ]
      }
    },

I have the exporter installed. I see high memory usage in all data nodes. 5 of 16 data nodes hit the 100% mem usage, the rest are 95%+


Heap usage looks not that bad

And the cluster is indexing 20k messages per second. We have 3 primary shards and 3 replica shards for each index.

Please let me how how to further improve the performance of OpenSearch

Hi @deng47 ,

Thanks a lot for the details.

But in the JVM output below, it shows both the default 1g heap size and my custom heap size in the input_arguments list. So I am not sure if that’s the cause of the performance issue.

It looks like the memory settings come from two different places, may be jvm.options + JAVA_OPTS / OPENSEARCH_JAVA_OPTS, but the heap stats indicates that the last setting wins (so you indeed have 2g / 4g / 8g heaps).

From the profile data, is seems like the query itself (data range) is taking a significant time, ~8 seconds for the one I see in the profile fragment. Could you please share what are the shards size [1] for this index?

[1] cat shards - OpenSearch documentation

Thank you for looking into this. @reta
I have 3 primary and 3 replica shards. I set the rollover shard size to 48g, and later 32g

my-system-syslog-000010 2 r STARTED 280328717   32gb 10.42.8.59   my-system-os-nodes-9
my-system-syslog-000010 2 p STARTED 280328717   32gb 10.42.18.23  my-system-os-nodes-15
my-system-syslog-000010 1 p STARTED 280332283   32gb 10.42.4.152  my-system-os-nodes-7
my-system-syslog-000010 1 r STARTED 280332283 32.1gb 10.42.7.159  my-system-os-nodes-1
my-system-syslog-000010 0 p STARTED 280312162   32gb 10.42.12.149 my-system-os-nodes-0
my-system-syslog-000010 0 r STARTED 280312162 32.1gb 10.42.9.32   my-system-os-nodes-12
my-system-syslog-000021 2 p STARTED 289305738 31.9gb 10.42.4.152  my-system-os-nodes-7
my-system-syslog-000021 2 r STARTED 289305738   32gb 10.42.18.23  my-system-os-nodes-15
my-system-syslog-000021 1 p STARTED 289334633 31.9gb 10.42.12.149 my-system-os-nodes-0
my-system-syslog-000021 1 r STARTED 289334633   32gb 10.42.7.159  my-system-os-nodes-1
my-system-syslog-000021 0 p STARTED 289307907 31.9gb 10.42.3.38   my-system-os-nodes-2
my-system-syslog-000021 0 r STARTED 289307907   32gb 10.42.9.32   my-system-os-nodes-12
my-system-syslog-000003 2 r STARTED 283667095 32.1gb 10.42.8.59   my-system-os-nodes-9
my-system-syslog-000003 2 p STARTED 283667095   32gb 10.42.18.23  my-system-os-nodes-15
my-system-syslog-000003 1 p STARTED 283671222   32gb 10.42.4.152  my-system-os-nodes-7
my-system-syslog-000003 1 r STARTED 283671222 32.1gb 10.42.7.159  my-system-os-nodes-1
my-system-syslog-000003 0 r STARTED 283675241   32gb 10.42.16.236 my-system-os-nodes-5
my-system-syslog-000003 0 p STARTED 283675241   32gb 10.42.9.32   my-system-os-nodes-12
my-system-syslog-000020 2 p STARTED 324230494 31.9gb 10.42.3.38   my-system-os-nodes-2
my-system-syslog-000020 2 r STARTED 324230494   32gb 10.42.18.23  my-system-os-nodes-15
my-system-syslog-000020 1 p STARTED 324217178 31.9gb 10.42.6.32   my-system-os-nodes-10
my-system-syslog-000020 1 r STARTED 324217178 31.9gb 10.42.4.152  my-system-os-nodes-7
my-system-syslog-000020 0 p STARTED 324231023 31.9gb 10.42.13.41  my-system-os-nodes-13
my-system-syslog-000020 0 r STARTED 324231023 31.9gb 10.42.12.149 my-system-os-nodes-0
my-system-syslog-000006 2 p STARTED 293905952 31.9gb 10.42.5.30   my-system-os-nodes-11
my-system-syslog-000006 2 r STARTED 293905952   32gb 10.42.12.149 my-system-os-nodes-0
my-system-syslog-000006 1 p STARTED 293922298 31.9gb 10.42.14.36  my-system-os-nodes-14
my-system-syslog-000006 1 r STARTED 293922298 31.9gb 10.42.4.152  my-system-os-nodes-7
my-system-syslog-000006 0 p STARTED 293913574 31.9gb 10.42.17.46  my-system-os-nodes-8
my-system-syslog-000006 0 r STARTED 293913574 31.9gb 10.42.3.38   my-system-os-nodes-2
my-system-syslog-000018 2 p STARTED 280630021   32gb 10.42.5.30   my-system-os-nodes-11
my-system-syslog-000018 2 r STARTED 280630021   32gb 10.42.3.38   my-system-os-nodes-2
my-system-syslog-000018 1 p STARTED 280692985   32gb 10.42.14.36  my-system-os-nodes-14
my-system-syslog-000018 1 r STARTED 280692985   32gb 10.42.12.149 my-system-os-nodes-0
my-system-syslog-000018 0 p STARTED 280680458   32gb 10.42.17.46  my-system-os-nodes-8
my-system-syslog-000018 0 r STARTED 280680458   32gb 10.42.13.41  my-system-os-nodes-13
my-system-syslog-000022 2 p STARTED 278755680 31.9gb 10.42.15.149 my-system-os-nodes-4
my-system-syslog-000022 2 r STARTED 278755680   32gb 10.42.17.46  my-system-os-nodes-8
my-system-syslog-000022 1 p STARTED 278764717   32gb 10.42.16.236 my-system-os-nodes-5
my-system-syslog-000022 1 r STARTED 278764717 32.1gb 10.42.14.36  my-system-os-nodes-14
my-system-syslog-000022 0 p STARTED 278748851   32gb 10.42.8.59   my-system-os-nodes-9
my-system-syslog-000022 0 r STARTED 278748851   32gb 10.42.10.36  my-system-os-nodes-3
my-system-syslog-000008 2 p STARTED 280351223 31.9gb 10.42.12.149 my-system-os-nodes-0
my-system-syslog-000008 2 r STARTED 280351223   32gb 10.42.9.32   my-system-os-nodes-12
my-system-syslog-000008 1 p STARTED 280350853 31.9gb 10.42.3.38   my-system-os-nodes-2
my-system-syslog-000008 1 r STARTED 280350853   32gb 10.42.18.23  my-system-os-nodes-15
my-system-syslog-000008 0 p STARTED 280362372 31.9gb 10.42.6.32   my-system-os-nodes-10
my-system-syslog-000008 0 r STARTED 280362372   32gb 10.42.4.152  my-system-os-nodes-7
my-system-syslog-000013 2 p STARTED 299377412 31.9gb 10.42.6.32   my-system-os-nodes-10
my-system-syslog-000013 2 r STARTED 299377412   32gb 10.42.4.152  my-system-os-nodes-7
my-system-syslog-000013 1 p STARTED 299367372 31.9gb 10.42.13.41  my-system-os-nodes-13
my-system-syslog-000013 1 r STARTED 299367372   32gb 10.42.12.149 my-system-os-nodes-0
my-system-syslog-000013 0 p STARTED 299378807 31.9gb 10.42.5.30   my-system-os-nodes-11
my-system-syslog-000013 0 r STARTED 299378807 31.9gb 10.42.3.38   my-system-os-nodes-2
my-system-syslog-000014 2 p STARTED 270741347 32.1gb 10.42.18.23  my-system-os-nodes-15
my-system-syslog-000014 2 r STARTED 270741347 32.1gb 10.42.7.159  my-system-os-nodes-1
my-system-syslog-000014 1 r STARTED 270754153   32gb 10.42.16.236 my-system-os-nodes-5
my-system-syslog-000014 1 p STARTED 270754153   32gb 10.42.12.149 my-system-os-nodes-0
my-system-syslog-000014 0 p STARTED 270770573   32gb 10.42.4.152  my-system-os-nodes-7
my-system-syslog-000014 0 r STARTED 270770573 32.1gb 10.42.9.32   my-system-os-nodes-12
my-system-syslog-000001 2 r STARTED 422760972   48gb 10.42.16.236 my-system-os-nodes-5
my-system-syslog-000001 2 p STARTED 422760972   48gb 10.42.4.152  my-system-os-nodes-7
my-system-syslog-000001 1 p STARTED 422796304   48gb 10.42.8.59   my-system-os-nodes-9
my-system-syslog-000001 1 r STARTED 422796304 48.1gb 10.42.10.36  my-system-os-nodes-3
my-system-syslog-000001 0 p STARTED 422764905   48gb 10.42.11.45  my-system-os-nodes-6
my-system-syslog-000001 0 r STARTED 422764905 48.1gb 10.42.15.149 my-system-os-nodes-4
my-system-syslog-000025 2 r STARTED 287711881 32.1gb 10.42.8.59   my-system-os-nodes-9
my-system-syslog-000025 2 p STARTED 287711881   32gb 10.42.18.23  my-system-os-nodes-15
my-system-syslog-000025 1 p STARTED 287744437   32gb 10.42.12.149 my-system-os-nodes-0
my-system-syslog-000025 1 r STARTED 287744437 32.1gb 10.42.7.159  my-system-os-nodes-1
my-system-syslog-000025 0 r STARTED 287740385 32.1gb 10.42.11.45  my-system-os-nodes-6
my-system-syslog-000025 0 p STARTED 287740385   32gb 10.42.4.152  my-system-os-nodes-7
my-system-syslog-000024 2 p STARTED 284584026 31.8gb 10.42.10.36  my-system-os-nodes-3
my-system-syslog-000024 2 r STARTED 284584026 31.9gb 10.42.5.30   my-system-os-nodes-11
my-system-syslog-000024 1 p STARTED 284540401 31.8gb 10.42.15.149 my-system-os-nodes-4
my-system-syslog-000024 1 r STARTED 284540401   32gb 10.42.14.36  my-system-os-nodes-14
my-system-syslog-000024 0 p STARTED 284546910 31.9gb 10.42.16.236 my-system-os-nodes-5
my-system-syslog-000024 0 r STARTED 284546910   32gb 10.42.17.46  my-system-os-nodes-8
my-system-syslog-000002 2 r STARTED 275128671 31.9gb 10.42.17.46  my-system-os-nodes-8
my-system-syslog-000002 2 p STARTED 275128671   32gb 10.42.13.41  my-system-os-nodes-13
my-system-syslog-000002 1 r STARTED 275104886 31.9gb 10.42.10.36  my-system-os-nodes-3
my-system-syslog-000002 1 p STARTED 275104886   32gb 10.42.5.30   my-system-os-nodes-11
my-system-syslog-000002 0 r STARTED 275130490 31.9gb 10.42.15.149 my-system-os-nodes-4
my-system-syslog-000002 0 p STARTED 275130490   32gb 10.42.14.36  my-system-os-nodes-14
my-system-syslog-000007 2 p STARTED 294167400   32gb 10.42.11.45  my-system-os-nodes-6
my-system-syslog-000007 2 r STARTED 294167400 32.1gb 10.42.10.36  my-system-os-nodes-3
my-system-syslog-000007 1 p STARTED 294121942   32gb 10.42.8.59   my-system-os-nodes-9
my-system-syslog-000007 1 r STARTED 294121942 32.1gb 10.42.15.149 my-system-os-nodes-4
my-system-syslog-000007 0 r STARTED 294107818 32.1gb 10.42.17.46  my-system-os-nodes-8
my-system-syslog-000007 0 p STARTED 294107818   32gb 10.42.9.32   my-system-os-nodes-12
my-system-syslog-000019 2 p STARTED 286880091   32gb 10.42.11.45  my-system-os-nodes-6
my-system-syslog-000019 2 r STARTED 286880091   32gb 10.42.15.149 my-system-os-nodes-4
my-system-syslog-000019 1 r STARTED 286848849   32gb 10.42.16.236 my-system-os-nodes-5
my-system-syslog-000019 1 p STARTED 286848849 31.9gb 10.42.9.32   my-system-os-nodes-12
my-system-syslog-000019 0 r STARTED 286840190 32.1gb 10.42.8.59   my-system-os-nodes-9
my-system-syslog-000019 0 p STARTED 286840190 31.9gb 10.42.18.23  my-system-os-nodes-15
my-system-syslog-000016 2 p STARTED 265531237 32.1gb 10.42.17.46  my-system-os-nodes-8
my-system-syslog-000016 2 r STARTED 265531237 32.2gb 10.42.3.38   my-system-os-nodes-2
my-system-syslog-000016 1 p STARTED 265548632 32.1gb 10.42.14.36  my-system-os-nodes-14
my-system-syslog-000016 1 r STARTED 265548632 32.2gb 10.42.13.41  my-system-os-nodes-13
my-system-syslog-000016 0 p STARTED 265543562 32.1gb 10.42.10.36  my-system-os-nodes-3
my-system-syslog-000016 0 r STARTED 265543562 32.2gb 10.42.5.30   my-system-os-nodes-11
my-system-syslog-000009 2 p STARTED 284744547 31.9gb 10.42.16.236 my-system-os-nodes-5
my-system-syslog-000009 2 r STARTED 284744547   32gb 10.42.13.41  my-system-os-nodes-13
my-system-syslog-000009 1 p STARTED 284739579 31.9gb 10.42.10.36  my-system-os-nodes-3
my-system-syslog-000009 1 r STARTED 284739579   32gb 10.42.5.30   my-system-os-nodes-11
my-system-syslog-000009 0 p STARTED 284747927   32gb 10.42.15.149 my-system-os-nodes-4
my-system-syslog-000009 0 r STARTED 284747927   32gb 10.42.14.36  my-system-os-nodes-14
my-system-syslog-000015 2 r STARTED 264799727 32.1gb 10.42.11.45  my-system-os-nodes-6
my-system-syslog-000015 2 p STARTED 264799727 32.1gb 10.42.18.23  my-system-os-nodes-15
my-system-syslog-000015 1 p STARTED 264825580 32.1gb 10.42.12.149 my-system-os-nodes-0
my-system-syslog-000015 1 r STARTED 264825580 32.1gb 10.42.9.32   my-system-os-nodes-12
my-system-syslog-000015 0 p STARTED 264802040 32.1gb 10.42.4.152  my-system-os-nodes-7
my-system-syslog-000015 0 r STARTED 264802040 32.1gb 10.42.7.159  my-system-os-nodes-1
my-system-syslog-000011 2 p STARTED 297509717   32gb 10.42.14.36  my-system-os-nodes-14
my-system-syslog-000011 2 r STARTED 297509717 32.1gb 10.42.6.32   my-system-os-nodes-10
my-system-syslog-000011 1 p STARTED 297551028   32gb 10.42.17.46  my-system-os-nodes-8
my-system-syslog-000011 1 r STARTED 297551028 32.1gb 10.42.13.41  my-system-os-nodes-13
my-system-syslog-000011 0 p STARTED 297561172   32gb 10.42.10.36  my-system-os-nodes-3
my-system-syslog-000011 0 r STARTED 297561172 32.1gb 10.42.5.30   my-system-os-nodes-11
my-system-syslog-000004 2 p STARTED 285692104 31.9gb 10.42.5.30   my-system-os-nodes-11
my-system-syslog-000004 2 r STARTED 285692104   32gb 10.42.3.38   my-system-os-nodes-2
my-system-syslog-000004 1 p STARTED 285650864 31.9gb 10.42.14.36  my-system-os-nodes-14
my-system-syslog-000004 1 r STARTED 285650864   32gb 10.42.6.32   my-system-os-nodes-10
my-system-syslog-000004 0 p STARTED 285696221 31.9gb 10.42.17.46  my-system-os-nodes-8
my-system-syslog-000004 0 r STARTED 285696221   32gb 10.42.13.41  my-system-os-nodes-13
my-system-syslog-000012 2 r STARTED 285865572   32gb 10.42.16.236 my-system-os-nodes-5
my-system-syslog-000012 2 p STARTED 285865572   32gb 10.42.7.159  my-system-os-nodes-1
my-system-syslog-000012 1 r STARTED 285865981   32gb 10.42.11.45  my-system-os-nodes-6
my-system-syslog-000012 1 p STARTED 285865981   32gb 10.42.9.32   my-system-os-nodes-12
my-system-syslog-000012 0 r STARTED 285850674   32gb 10.42.8.59   my-system-os-nodes-9
my-system-syslog-000012 0 p STARTED 285850674   32gb 10.42.18.23  my-system-os-nodes-15
my-system-syslog-000023 2 p STARTED 284333125 32.2gb 10.42.14.36  my-system-os-nodes-14
my-system-syslog-000023 2 r STARTED 284333125 32.2gb 10.42.13.41  my-system-os-nodes-13
my-system-syslog-000023 1 p STARTED 284283485 32.1gb 10.42.10.36  my-system-os-nodes-3
my-system-syslog-000023 1 r STARTED 284283485 32.2gb 10.42.5.30   my-system-os-nodes-11
my-system-syslog-000023 0 p STARTED 284290524 32.1gb 10.42.15.149 my-system-os-nodes-4
my-system-syslog-000023 0 r STARTED 284290524 32.2gb 10.42.17.46  my-system-os-nodes-8
my-system-syslog-000005 2 r STARTED 283428976   32gb 10.42.16.236 my-system-os-nodes-5
my-system-syslog-000005 2 p STARTED 283428976   32gb 10.42.9.32   my-system-os-nodes-12
my-system-syslog-000005 1 r STARTED 283429414   32gb 10.42.11.45  my-system-os-nodes-6
my-system-syslog-000005 1 p STARTED 283429414   32gb 10.42.7.159  my-system-os-nodes-1
my-system-syslog-000005 0 r STARTED 283430250   32gb 10.42.8.59   my-system-os-nodes-9
my-system-syslog-000005 0 p STARTED 283430250   32gb 10.42.18.23  my-system-os-nodes-15
my-system-syslog-000017 2 r STARTED 263422724 32.1gb 10.42.11.45  my-system-os-nodes-6
my-system-syslog-000017 2 p STARTED 263422724   32gb 10.42.4.152  my-system-os-nodes-7
my-system-syslog-000017 1 r STARTED 263398695 32.1gb 10.42.8.59   my-system-os-nodes-9
my-system-syslog-000017 1 p STARTED 263398695   32gb 10.42.12.149 my-system-os-nodes-0
my-system-syslog-000017 0 r STARTED 263385057   32gb 10.42.16.236 my-system-os-nodes-5
my-system-syslog-000017 0 p STARTED 263385057 32.1gb 10.42.18.23  my-system-os-nodes-15
my-system-syslog-000027 2 p STARTED 139255010 16.2gb 10.42.14.36  my-system-os-nodes-14
my-system-syslog-000027 2 r STARTED 139218435 16.2gb 10.42.13.41  my-system-os-nodes-13
my-system-syslog-000027 1 p STARTED 139273701 16.5gb 10.42.10.36  my-system-os-nodes-3
my-system-syslog-000027 1 r STARTED 139216672 16.3gb 10.42.5.30   my-system-os-nodes-11
my-system-syslog-000027 0 p STARTED 139223567 16.7gb 10.42.15.149 my-system-os-nodes-4
my-system-syslog-000027 0 r STARTED 139240000 16.2gb 10.42.17.46  my-system-os-nodes-8
my-system-syslog-000026 2 p STARTED 282303985   32gb 10.42.12.149 my-system-os-nodes-0
my-system-syslog-000026 2 r STARTED 282303985 32.1gb 10.42.9.32   my-system-os-nodes-12
my-system-syslog-000026 1 p STARTED 282309315 32.1gb 10.42.4.152  my-system-os-nodes-7
my-system-syslog-000026 1 r STARTED 282309315 32.1gb 10.42.7.159  my-system-os-nodes-1
my-system-syslog-000026 0 p STARTED 282302952   32gb 10.42.3.38   my-system-os-nodes-2
my-system-syslog-000026 0 r STARTED 282302952 32.1gb 10.42.18.23  my-system-os-nodes-15

Thanks @deng47, the shards size / distribution is looking good, as per rollover policy configured. I assume the range query for 24 should only impact a few indices (based on the date range), not all of them.

May I ask you please, if possible, to try just search (without aggregation) and share the diff you see between OpenSearch and Elasticsearch with profiling (trying to narrow the problem), so we could pinpoint which stage is slower.

Thank you.

Unfortunately, my ElasticSearch cluster is down, so I can’t provide the ES profiling result.
Searching data (without aggregation) took 19 seconds. I will paste the result in multiple blocks due to len limit

GET /my-system-syslog/_search
{
  "profile": true,
  "version": true,
  "size": 500,
  "sort": [
    {
      "timestamp": {
        "order": "desc",
        "unmapped_type": "boolean"
      }
    }
  ],
  "stored_fields": [
    "*"
  ],
  "docvalue_fields": [
    {
      "field": "@timestamp",
      "format": "date_time"
    },
    {
      "field": "timestamp",
      "format": "date_time"
    }
  ],
  "_source": {
    "excludes": []
  },
  "query": {
    "bool": {
      "must": [],
      "filter": [
        {
          "match_all": {}
        },
        {
          "range": {
            "timestamp": {              
              "gte": "2023-01-20T04:00:00.000Z",
              "lte": "2023-01-21T04:00:00.000Z",
              "format": "strict_date_optional_time"
            }
          }
        }
      ],
      "should": [],
      "must_not": []
    }
  }
}
{
  "took" : 19391,
  "timed_out" : false,
  "_shards" : {
    "total" : 81,
    "successful" : 81,
    "skipped" : 36,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10000,
      "relation" : "gte"
    },
    "max_score" : null,
    "hits" : <SKIP ALL RESULTS>
      },
  "profile" : {
    "shards" : [
      {
        "id" : "[5pwR-7jcQraOj--MGQVrjw][my-system-syslog-000012][1]",
        "inbound_network_time_in_millis" : 1,
        "outbound_network_time_in_millis" : 1,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 210239532,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 7750,
                  "advance_count" : 33,
                  "score" : 0,
                  "build_scorer_count" : 66,
                  "create_weight" : 26740,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 210205042
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 210147862,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 4180,
                      "advance_count" : 33,
                      "score" : 0,
                      "build_scorer_count" : 66,
                      "create_weight" : 3170,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 210140512
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 32780,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 493322
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[5pwR-7jcQraOj--MGQVrjw][my-system-syslog-000015][1]",
        "inbound_network_time_in_millis" : 2,
        "outbound_network_time_in_millis" : 1,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 9034300943,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 8762824242,
                  "match" : 0,
                  "next_doc_count" : 112080564,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 12120,
                  "advance_count" : 36,
                  "score" : 0,
                  "build_scorer_count" : 72,
                  "create_weight" : 9680,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 271454901
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 3219811480,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 2948381579,
                      "match" : 0,
                      "next_doc_count" : 112080564,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 9050,
                      "advance_count" : 36,
                      "score" : 0,
                      "build_scorer_count" : 72,
                      "create_weight" : 610,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 271420241
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 12830,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 4446570194
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[5pwR-7jcQraOj--MGQVrjw][my-system-syslog-000026][2]",
        "inbound_network_time_in_millis" : 1,
        "outbound_network_time_in_millis" : 1,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 358314542,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 9900,
                  "advance_count" : 35,
                  "score" : 0,
                  "build_scorer_count" : 70,
                  "create_weight" : 16760,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 358287882
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 358269763,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 6160,
                      "advance_count" : 35,
                      "score" : 0,
                      "build_scorer_count" : 70,
                      "create_weight" : 1830,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 358261773
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 25290,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 487543
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[6brGb5iXQkuT9g-r-7gmLQ][my-system-syslog-000001][1]",
        "inbound_network_time_in_millis" : 2,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 85398226,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 9380,
                  "advance_count" : 32,
                  "score" : 0,
                  "build_scorer_count" : 64,
                  "create_weight" : 17870,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 85370976
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 85357766,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 4470,
                      "advance_count" : 32,
                      "score" : 0,
                      "build_scorer_count" : 64,
                      "create_weight" : 2020,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 85351276
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 31920,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 397882
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[6brGb5iXQkuT9g-r-7gmLQ][my-system-syslog-000022][0]",
        "inbound_network_time_in_millis" : 1,
        "outbound_network_time_in_millis" : 2,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 22100033,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 8690,
                  "advance_count" : 25,
                  "score" : 0,
                  "build_scorer_count" : 50,
                  "create_weight" : 51339,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 22040004
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 22031995,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 4050,
                      "advance_count" : 25,
                      "score" : 0,
                      "build_scorer_count" : 50,
                      "create_weight" : 3460,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 22024485
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 46209,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 746268
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[DD0Tc2xHS2qCuzDfQQ9njA][my-system-syslog-000013][1]",
        "inbound_network_time_in_millis" : 3,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 12227487543,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 12076470668,
                  "match" : 0,
                  "next_doc_count" : 153439092,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 38819,
                  "advance_count" : 35,
                  "score" : 0,
                  "build_scorer_count" : 70,
                  "create_weight" : 22659,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 150955397
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 4180414725,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 4029450358,
                      "match" : 0,
                      "next_doc_count" : 153439092,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 33859,
                      "advance_count" : 35,
                      "score" : 0,
                      "build_scorer_count" : 70,
                      "create_weight" : 3050,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 150927458
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 19720,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 6244170237
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[IHm5ZVmkR4KH1p6kcQZUoQ][my-system-syslog-000004][1]",
        "inbound_network_time_in_millis" : 0,
        "outbound_network_time_in_millis" : 9,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 162839298,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 13970,
                  "advance_count" : 29,
                  "score" : 0,
                  "build_scorer_count" : 58,
                  "create_weight" : 32499,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 162792829
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 162775769,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 8750,
                      "advance_count" : 29,
                      "score" : 0,
                      "build_scorer_count" : 58,
                      "create_weight" : 2630,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 162764389
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 35930,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 457535
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[IHm5ZVmkR4KH1p6kcQZUoQ][my-system-syslog-000006][1]",
        "inbound_network_time_in_millis" : 0,
        "outbound_network_time_in_millis" : 1,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 628642672,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 29730,
                  "advance_count" : 35,
                  "score" : 0,
                  "build_scorer_count" : 70,
                  "create_weight" : 10189,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 628602753
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 628577103,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 21880,
                      "advance_count" : 35,
                      "score" : 0,
                      "build_scorer_count" : 70,
                      "create_weight" : 610,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 628554613
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 14800,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 692768
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[IHm5ZVmkR4KH1p6kcQZUoQ][my-system-syslog-000016][1]",
        "inbound_network_time_in_millis" : 0,
        "outbound_network_time_in_millis" : 1,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 620611635,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 22759,
                  "match" : 0,
                  "next_doc_count" : 115,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 30329,
                  "advance_count" : 37,
                  "score" : 0,
                  "build_scorer_count" : 74,
                  "create_weight" : 22450,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 620536097
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 620522747,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 9579,
                      "match" : 0,
                      "next_doc_count" : 115,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 23390,
                      "advance_count" : 37,
                      "score" : 0,
                      "build_scorer_count" : 74,
                      "create_weight" : 2130,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 620487648
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 20630,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 38850575
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[IHm5ZVmkR4KH1p6kcQZUoQ][my-system-syslog-000023][2]",
        "inbound_network_time_in_millis" : 1,
        "outbound_network_time_in_millis" : 2,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 654012908,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 25650,
                  "advance_count" : 32,
                  "score" : 0,
                  "build_scorer_count" : 64,
                  "create_weight" : 10480,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 653976778
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 653955638,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 18560,
                      "advance_count" : 32,
                      "score" : 0,
                      "build_scorer_count" : 64,
                      "create_weight" : 1130,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 653935948
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 12960,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 684654
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[IHm5ZVmkR4KH1p6kcQZUoQ][my-system-syslog-000027][2]",
        "inbound_network_time_in_millis" : 0,
        "outbound_network_time_in_millis" : 1,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 262146421,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 12739,
                  "advance_count" : 42,
                  "score" : 0,
                  "build_scorer_count" : 84,
                  "create_weight" : 19580,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 262114102
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 262104192,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 7560,
                      "advance_count" : 42,
                      "score" : 0,
                      "build_scorer_count" : 84,
                      "create_weight" : 2670,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 262093962
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 28699,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 359975
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[NSBvJvPfT-aWaKelQj8E1A][my-system-syslog-000004][0]",
        "inbound_network_time_in_millis" : 0,
        "outbound_network_time_in_millis" : 8,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 522623,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 17710,
                  "advance_count" : 28,
                  "score" : 0,
                  "build_scorer_count" : 56,
                  "create_weight" : 55879,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 449034
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 444096,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 2430,
                      "advance_count" : 28,
                      "score" : 0,
                      "build_scorer_count" : 56,
                      "create_weight" : 22160,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 419506
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 123119,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 509000
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[OPt3-GOGRJaWy3vfphawZw][my-system-syslog-000004][2]",
        "inbound_network_time_in_millis" : 1,
        "outbound_network_time_in_millis" : 4,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 297633,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 7210,
                  "advance_count" : 34,
                  "score" : 0,
                  "build_scorer_count" : 68,
                  "create_weight" : 22789,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 267634
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 254775,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 2700,
                      "advance_count" : 34,
                      "score" : 0,
                      "build_scorer_count" : 68,
                      "create_weight" : 1520,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 250555
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 35899,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 504710
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[OPt3-GOGRJaWy3vfphawZw][my-system-syslog-000006][0]",
        "inbound_network_time_in_millis" : 1,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 273181280,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 12050,
                  "advance_count" : 33,
                  "score" : 0,
                  "build_scorer_count" : 66,
                  "create_weight" : 22569,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 273146661
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 273117871,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 6220,
                      "advance_count" : 33,
                      "score" : 0,
                      "build_scorer_count" : 66,
                      "create_weight" : 1470,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 273110181
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 27080,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 680316
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[OPt3-GOGRJaWy3vfphawZw][my-system-syslog-000013][0]",
        "inbound_network_time_in_millis" : 2,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 12391602916,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 12028231127,
                  "match" : 0,
                  "next_doc_count" : 153421824,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 31200,
                  "advance_count" : 34,
                  "score" : 0,
                  "build_scorer_count" : 68,
                  "create_weight" : 16980,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 363323609
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 4349976207,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 3986641998,
                      "match" : 0,
                      "next_doc_count" : 153421824,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 28470,
                      "advance_count" : 34,
                      "score" : 0,
                      "build_scorer_count" : 68,
                      "create_weight" : 1280,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 363304459
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 28980,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 6050888257
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[OPt3-GOGRJaWy3vfphawZw][my-system-syslog-000016][2]",
        "inbound_network_time_in_millis" : 3,
        "outbound_network_time_in_millis" : 1,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 143626324,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 20460,
                  "match" : 0,
                  "next_doc_count" : 83,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 31819,
                  "advance_count" : 32,
                  "score" : 0,
                  "build_scorer_count" : 64,
                  "create_weight" : 14040,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 143560005
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 143574704,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 9530,
                      "match" : 0,
                      "next_doc_count" : 83,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 26229,
                      "advance_count" : 32,
                      "score" : 0,
                      "build_scorer_count" : 64,
                      "create_weight" : 2120,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 143536825
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 16380,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 32355443
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[OPt3-GOGRJaWy3vfphawZw][my-system-syslog-000026][0]",
        "inbound_network_time_in_millis" : 1,
        "outbound_network_time_in_millis" : 2,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 371263,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 5900,
                  "advance_count" : 28,
                  "score" : 0,
                  "build_scorer_count" : 56,
                  "create_weight" : 11050,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 354313
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 343044,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 2100,
                      "advance_count" : 28,
                      "score" : 0,
                      "build_scorer_count" : 56,
                      "create_weight" : 750,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 340194
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 23098,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 92658
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[_GhfQCzFSkW8pmt3mAeUwA][my-system-syslog-000001][0]",
        "inbound_network_time_in_millis" : 0,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 460712,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 10090,
                  "advance_count" : 39,
                  "score" : 0,
                  "build_scorer_count" : 78,
                  "create_weight" : 18350,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 432272
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 418472,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 4890,
                      "advance_count" : 39,
                      "score" : 0,
                      "build_scorer_count" : 78,
                      "create_weight" : 3960,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 409622
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 43739,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 443544
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[_GhfQCzFSkW8pmt3mAeUwA][my-system-syslog-000019][2]",
        "inbound_network_time_in_millis" : 0,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 237708404,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 1420,
                  "match" : 0,
                  "next_doc_count" : 1,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 9038,
                  "advance_count" : 32,
                  "score" : 0,
                  "build_scorer_count" : 64,
                  "create_weight" : 13860,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 237684086
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 237665884,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 1250,
                      "match" : 0,
                      "next_doc_count" : 1,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 6788,
                      "advance_count" : 32,
                      "score" : 0,
                      "build_scorer_count" : 64,
                      "create_weight" : 2190,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 237655656
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 23649,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 41635544
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[_GhfQCzFSkW8pmt3mAeUwA][my-system-syslog-000022][2]",
        "inbound_network_time_in_millis" : 0,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 58618438,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 6470,
                  "advance_count" : 29,
                  "score" : 0,
                  "build_scorer_count" : 58,
                  "create_weight" : 9679,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 58602289
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 58591510,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 3100,
                      "advance_count" : 29,
                      "score" : 0,
                      "build_scorer_count" : 58,
                      "create_weight" : 1070,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 58587340
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 15400,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 148127
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[_GhfQCzFSkW8pmt3mAeUwA][my-system-syslog-000027][0]",
        "inbound_network_time_in_millis" : 0,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 169569537,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 10950,
                  "advance_count" : 55,
                  "score" : 0,
                  "build_scorer_count" : 110,
                  "create_weight" : 15220,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 169543367
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 169525208,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 4930,
                      "advance_count" : 55,
                      "score" : 0,
                      "build_scorer_count" : 110,
                      "create_weight" : 2730,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 169517548
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 28119,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 206807
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[bifQA3XVQQKuI_EMhxyGlg][my-system-syslog-000023][0]",
        "inbound_network_time_in_millis" : 1,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 137051163,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 10120,
                  "advance_count" : 33,
                  "score" : 0,
                  "build_scorer_count" : 66,
                  "create_weight" : 30470,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 137010573
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 136980024,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 4760,
                      "advance_count" : 33,
                      "score" : 0,
                      "build_scorer_count" : 66,
                      "create_weight" : 2010,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 136973254
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 66739,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 678881
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[dYwPIUy0SdGb36b4HCl7yw][my-system-syslog-000003][1]",
        "inbound_network_time_in_millis" : 8,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 341443,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 3800,
                  "advance_count" : 31,
                  "score" : 0,
                  "build_scorer_count" : 62,
                  "create_weight" : 11760,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 325883
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 321823,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 1640,
                      "advance_count" : 31,
                      "score" : 0,
                      "build_scorer_count" : 62,
                      "create_weight" : 2270,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 317913
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 20360,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 178806
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[dYwPIUy0SdGb36b4HCl7yw][my-system-syslog-000005][1]",
        "inbound_network_time_in_millis" : 10,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 371193,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 4580,
                  "advance_count" : 21,
                  "score" : 0,
                  "build_scorer_count" : 42,
                  "create_weight" : 14440,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 352173
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 347583,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 1780,
                      "advance_count" : 21,
                      "score" : 0,
                      "build_scorer_count" : 42,
                      "create_weight" : 2480,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 343323
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 22170,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 181966
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[dYwPIUy0SdGb36b4HCl7yw][my-system-syslog-000015][0]",
        "inbound_network_time_in_millis" : 7,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 9011562921,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 8823178463,
                  "match" : 0,
                  "next_doc_count" : 112091986,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 11080,
                  "advance_count" : 35,
                  "score" : 0,
                  "build_scorer_count" : 70,
                  "create_weight" : 15400,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 188357978
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 3152306717,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 2963958628,
                      "match" : 0,
                      "next_doc_count" : 112091986,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 7390,
                      "advance_count" : 35,
                      "score" : 0,
                      "build_scorer_count" : 70,
                      "create_weight" : 2570,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 188338129
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 23479,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 4190731490
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[dYwPIUy0SdGb36b4HCl7yw][my-system-syslog-000026][1]",
        "inbound_network_time_in_millis" : 6,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 53118702,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 10730,
                  "advance_count" : 35,
                  "score" : 0,
                  "build_scorer_count" : 70,
                  "create_weight" : 41969,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 53066003
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 53067383,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 4690,
                      "advance_count" : 35,
                      "score" : 0,
                      "build_scorer_count" : 70,
                      "create_weight" : 14700,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 53047993
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 87678,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 621557
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[g1LncdRoRCez4uKHcw12YQ][my-system-syslog-000001][2]",
        "inbound_network_time_in_millis" : 1,
        "outbound_network_time_in_millis" : 1,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 314844,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 9340,
                  "advance_count" : 36,
                  "score" : 0,
                  "build_scorer_count" : 72,
                  "create_weight" : 9600,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 295904
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 286184,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 4400,
                      "advance_count" : 36,
                      "score" : 0,
                      "build_scorer_count" : 72,
                      "create_weight" : 820,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 280964
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 21570,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 156169
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[g1LncdRoRCez4uKHcw12YQ][my-system-syslog-000003][0]",
        "inbound_network_time_in_millis" : 1,
        "outbound_network_time_in_millis" : 2,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 378214,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 11360,
                  "advance_count" : 35,
                  "score" : 0,
                  "build_scorer_count" : 70,
                  "create_weight" : 11430,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 355424
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 341184,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 5790,
                      "advance_count" : 35,
                      "score" : 0,
                      "build_scorer_count" : 70,
                      "create_weight" : 1150,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 334244
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 24270,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 128707
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },

      {
        "id" : "[g1LncdRoRCez4uKHcw12YQ][my-system-syslog-000005][2]",
        "inbound_network_time_in_millis" : 0,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 285804,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 10110,
                  "advance_count" : 29,
                  "score" : 0,
                  "build_scorer_count" : 58,
                  "create_weight" : 9740,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 265954
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 240685,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 6130,
                      "advance_count" : 29,
                      "score" : 0,
                      "build_scorer_count" : 58,
                      "create_weight" : 930,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 233625
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 21740,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 127249
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[g1LncdRoRCez4uKHcw12YQ][my-system-syslog-000012][2]",
        "inbound_network_time_in_millis" : 1,
        "outbound_network_time_in_millis" : 3,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 472899,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 11980,
                  "advance_count" : 32,
                  "score" : 0,
                  "build_scorer_count" : 64,
                  "create_weight" : 25739,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 435180
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 423951,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 6650,
                      "advance_count" : 32,
                      "score" : 0,
                      "build_scorer_count" : 64,
                      "create_weight" : 5010,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 412291
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 83978,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 464703
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[g1LncdRoRCez4uKHcw12YQ][my-system-syslog-000014][1]",
        "inbound_network_time_in_millis" : 1,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 21350027562,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 21347103196,
                  "match" : 0,
                  "next_doc_count" : 270753955,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 10400,
                  "advance_count" : 34,
                  "score" : 0,
                  "build_scorer_count" : 68,
                  "create_weight" : 11470,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 2902496
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 7154279396,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 7151391890,
                      "match" : 0,
                      "next_doc_count" : 270753955,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 8030,
                      "advance_count" : 34,
                      "score" : 0,
                      "build_scorer_count" : 68,
                      "create_weight" : 1100,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 2878376
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 23010,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 9318409475
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[g1LncdRoRCez4uKHcw12YQ][my-system-syslog-000017][0]",
        "inbound_network_time_in_millis" : 0,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 315904,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 510,
                  "match" : 0,
                  "next_doc_count" : 1,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 12049,
                  "advance_count" : 37,
                  "score" : 0,
                  "build_scorer_count" : 74,
                  "create_weight" : 14350,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 288995
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 270775,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 360,
                      "match" : 0,
                      "next_doc_count" : 1,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 6909,
                      "advance_count" : 37,
                      "score" : 0,
                      "build_scorer_count" : 74,
                      "create_weight" : 1600,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 261906
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 34750,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 192797
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[g1LncdRoRCez4uKHcw12YQ][my-system-syslog-000022][1]",
        "inbound_network_time_in_millis" : 0,
        "outbound_network_time_in_millis" : 3,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 307272,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 8250,
                  "advance_count" : 30,
                  "score" : 0,
                  "build_scorer_count" : 60,
                  "create_weight" : 14639,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 284383
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 274973,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 4110,
                      "advance_count" : 30,
                      "score" : 0,
                      "build_scorer_count" : 60,
                      "create_weight" : 1690,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 269173
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 70479,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 225038
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[ol-92BbaSzaxHEDvX6FcXA][my-system-syslog-000003][2]",
        "inbound_network_time_in_millis" : 1,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 294196299,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 11370,
                  "advance_count" : 37,
                  "score" : 0,
                  "build_scorer_count" : 74,
                  "create_weight" : 10299,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 294174630
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 294145382,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 6390,
                      "advance_count" : 37,
                      "score" : 0,
                      "build_scorer_count" : 74,
                      "create_weight" : 830,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 294138162
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 13260,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 2787312
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[ol-92BbaSzaxHEDvX6FcXA][my-system-syslog-000005][0]",
        "inbound_network_time_in_millis" : 1,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 233541735,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 11380,
                  "advance_count" : 35,
                  "score" : 0,
                  "build_scorer_count" : 70,
                  "create_weight" : 18310,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 233512045
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 233500276,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 7000,
                      "advance_count" : 35,
                      "score" : 0,
                      "build_scorer_count" : 70,
                      "create_weight" : 3820,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 233489456
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 22610,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 1152189
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[ol-92BbaSzaxHEDvX6FcXA][my-system-syslog-000012][0]",
        "inbound_network_time_in_millis" : 1,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 439078852,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 12090,
                  "advance_count" : 32,
                  "score" : 0,
                  "build_scorer_count" : 64,
                  "create_weight" : 37700,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 439029062
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 439017843,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 7170,
                      "advance_count" : 32,
                      "score" : 0,
                      "build_scorer_count" : 64,
                      "create_weight" : 5890,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 439004783
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 46729,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 2111813
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[ol-92BbaSzaxHEDvX6FcXA][my-system-syslog-000014][2]",
        "inbound_network_time_in_millis" : 1,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 21216192751,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 21209265516,
                  "match" : 0,
                  "next_doc_count" : 270741156,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 11720,
                  "advance_count" : 33,
                  "score" : 0,
                  "build_scorer_count" : 66,
                  "create_weight" : 17520,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 6897995
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 7125201685,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 7118308120,
                      "match" : 0,
                      "next_doc_count" : 270741156,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 7110,
                      "advance_count" : 33,
                      "score" : 0,
                      "build_scorer_count" : 66,
                      "create_weight" : 2750,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 6883705
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 25600,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 10325655850
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[ol-92BbaSzaxHEDvX6FcXA][my-system-syslog-000015][2]",
        "inbound_network_time_in_millis" : 1,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 8999836189,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 8810136288,
                  "match" : 0,
                  "next_doc_count" : 112075842,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 8090,
                  "advance_count" : 30,
                  "score" : 0,
                  "build_scorer_count" : 60,
                  "create_weight" : 49719,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 189642092
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 3153102907,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 2963464785,
                      "match" : 0,
                      "next_doc_count" : 112075842,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 5360,
                      "advance_count" : 30,
                      "score" : 0,
                      "build_scorer_count" : 60,
                      "create_weight" : 2650,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 189630112
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 28330,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 4203000714
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[ol-92BbaSzaxHEDvX6FcXA][my-system-syslog-000020][2]",
        "inbound_network_time_in_millis" : 1,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 395301637,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 1850,
                  "match" : 0,
                  "next_doc_count" : 2,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 8860,
                  "advance_count" : 31,
                  "score" : 0,
                  "build_scorer_count" : 62,
                  "create_weight" : 17340,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 395273587
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 395260787,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 1360,
                      "match" : 0,
                      "next_doc_count" : 2,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 5120,
                      "advance_count" : 31,
                      "score" : 0,
                      "build_scorer_count" : 62,
                      "create_weight" : 2520,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 395251787
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 28170,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 180595584
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[pxKoHmErQI63ky1oZo4yXg][my-system-syslog-000013][2]",
        "inbound_network_time_in_millis" : 1,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 12063404189,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 12009752346,
                  "match" : 0,
                  "next_doc_count" : 153438820,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 8640,
                  "advance_count" : 27,
                  "score" : 0,
                  "build_scorer_count" : 54,
                  "create_weight" : 87979,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 53555224
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 4044706605,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 3991099082,
                      "match" : 0,
                      "next_doc_count" : 153438820,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 5600,
                      "advance_count" : 27,
                      "score" : 0,
                      "build_scorer_count" : 54,
                      "create_weight" : 62229,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 53539694
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 43950,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 6069404262
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[pxKoHmErQI63ky1oZo4yXg][my-system-syslog-000014][0]",
        "inbound_network_time_in_millis" : 1,
        "outbound_network_time_in_millis" : 1,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 21370639248,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 21367383614,
                  "match" : 0,
                  "next_doc_count" : 270770395,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 10310,
                  "advance_count" : 34,
                  "score" : 0,
                  "build_scorer_count" : 68,
                  "create_weight" : 64269,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 3181055
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 7104083495,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 7100917190,
                      "match" : 0,
                      "next_doc_count" : 270770395,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 6680,
                      "advance_count" : 34,
                      "score" : 0,
                      "build_scorer_count" : 68,
                      "create_weight" : 2390,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 3157235
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 41430,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 10709644881
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[yagqiHy7RIOEnmHgiEi2iw][my-system-syslog-000006][2]",
        "inbound_network_time_in_millis" : 1,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 234013883,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 18410,
                  "advance_count" : 33,
                  "score" : 0,
                  "build_scorer_count" : 66,
                  "create_weight" : 33130,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 233962343
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 233939644,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 11370,
                      "advance_count" : 33,
                      "score" : 0,
                      "build_scorer_count" : 66,
                      "create_weight" : 3360,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 233924914
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 50129,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 350663
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },


      {
        "id" : "[yagqiHy7RIOEnmHgiEi2iw][my-system-syslog-000016][0]",
        "inbound_network_time_in_millis" : 1,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 408896796,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 14640,
                  "match" : 0,
                  "next_doc_count" : 90,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 23350,
                  "advance_count" : 34,
                  "score" : 0,
                  "build_scorer_count" : 68,
                  "create_weight" : 15480,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 408843326
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 408826506,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 6670,
                      "match" : 0,
                      "next_doc_count" : 90,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 14510,
                      "advance_count" : 34,
                      "score" : 0,
                      "build_scorer_count" : 68,
                      "create_weight" : 1530,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 408803796
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 26609,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 68622265
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[yagqiHy7RIOEnmHgiEi2iw][my-system-syslog-000023][1]",
        "inbound_network_time_in_millis" : 3,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 402339183,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 17720,
                  "advance_count" : 30,
                  "score" : 0,
                  "build_scorer_count" : 60,
                  "create_weight" : 14359,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 402307104
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 402288744,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 11620,
                      "advance_count" : 30,
                      "score" : 0,
                      "build_scorer_count" : 60,
                      "create_weight" : 630,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 402276494
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 13460,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 631450
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      {
        "id" : "[yagqiHy7RIOEnmHgiEi2iw][my-system-syslog-000027][1]",
        "inbound_network_time_in_millis" : 1,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
                "time_in_nanos" : 61032496,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 0,
                  "match" : 0,
                  "next_doc_count" : 0,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 12200,
                  "advance_count" : 39,
                  "score" : 0,
                  "build_scorer_count" : 78,
                  "create_weight" : 33550,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 60986746
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674187200000 TO 1674273600000]",
                    "time_in_nanos" : 60966156,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 0,
                      "match" : 0,
                      "next_doc_count" : 0,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 6260,
                      "advance_count" : 39,
                      "score" : 0,
                      "build_scorer_count" : 78,
                      "create_weight" : 4140,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 60955756
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 31720,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 336696
              }
            ]
          }
        ],
        "aggregations" : [ ]
      }
    ]
  }
}

Hi @deng47 ,

Thanks for sharing profile data, it seems like the queries over my-system-syslog-000014 index dominate over others, taking more that 20s per shard. Could you correlate other metrics (like JVM heap usage fe) on the nodes g1LncdRoRCez4uKHcw12YQ, ol-92BbaSzaxHEDvX6FcXA and pxKoHmErQI63ky1oZo4yXg ?

  {
    "id" : "[g1LncdRoRCez4uKHcw12YQ][my-system-syslog-000014][1]",
    "inbound_network_time_in_millis" : 1,
    "outbound_network_time_in_millis" : 0,
    "searches" : [
      {
        "query" : [
          {
            "type" : "ConstantScoreQuery",
            "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
            "time_in_nanos" : 21350027562,
            "breakdown" : {
              "set_min_competitive_score_count" : 0,
              "match_count" : 0,
              "shallow_advance_count" : 0,
              "set_min_competitive_score" : 0,
              "next_doc" : 21347103196,
              "match" : 0,
              "next_doc_count" : 270753955,
              "score_count" : 0,
              "compute_max_score_count" : 0,
              "compute_max_score" : 0,
              "advance" : 10400,
              "advance_count" : 34,
              "score" : 0,
              "build_scorer_count" : 68,
              "create_weight" : 11470,
              "shallow_advance" : 0,
              "create_weight_count" : 1,
              "build_scorer" : 2902496
            },
            "children" : [
              {
                "type" : "IndexOrDocValuesQuery",
                "description" : "timestamp:[1674187200000 TO 1674273600000]",
                "time_in_nanos" : 7154279396,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 7151391890,
                  "match" : 0,
                  "next_doc_count" : 270753955,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 8030,
                  "advance_count" : 34,
                  "score" : 0,
                  "build_scorer_count" : 68,
                  "create_weight" : 1100,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 2878376
                }
              }
            ]
          }
        ],
        "rewrite_time" : 23010,
        "collector" : [
          {
            "name" : "SimpleFieldCollector",
            "reason" : "search_top_hits",
            "time_in_nanos" : 9318409475
          }
        ]
      }
    ],
    "aggregations" : [ ]
  },

  {
    "id" : "[ol-92BbaSzaxHEDvX6FcXA][my-system-syslog-000014][2]",
    "inbound_network_time_in_millis" : 1,
    "outbound_network_time_in_millis" : 0,
    "searches" : [
      {
        "query" : [
          {
            "type" : "ConstantScoreQuery",
            "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
            "time_in_nanos" : 21216192751,
            "breakdown" : {
              "set_min_competitive_score_count" : 0,
              "match_count" : 0,
              "shallow_advance_count" : 0,
              "set_min_competitive_score" : 0,
              "next_doc" : 21209265516,
              "match" : 0,
              "next_doc_count" : 270741156,
              "score_count" : 0,
              "compute_max_score_count" : 0,
              "compute_max_score" : 0,
              "advance" : 11720,
              "advance_count" : 33,
              "score" : 0,
              "build_scorer_count" : 66,
              "create_weight" : 17520,
              "shallow_advance" : 0,
              "create_weight_count" : 1,
              "build_scorer" : 6897995
            },
            "children" : [
              {
                "type" : "IndexOrDocValuesQuery",
                "description" : "timestamp:[1674187200000 TO 1674273600000]",
                "time_in_nanos" : 7125201685,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 7118308120,
                  "match" : 0,
                  "next_doc_count" : 270741156,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 7110,
                  "advance_count" : 33,
                  "score" : 0,
                  "build_scorer_count" : 66,
                  "create_weight" : 2750,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 6883705
                }
              }
            ]
          }
        ],
        "rewrite_time" : 25600,
        "collector" : [
          {
            "name" : "SimpleFieldCollector",
            "reason" : "search_top_hits",
            "time_in_nanos" : 10325655850
          }
        ]
      }
    ],
    "aggregations" : [ ]
  },

  {
    "id" : "[pxKoHmErQI63ky1oZo4yXg][my-system-syslog-000014][0]",
    "inbound_network_time_in_millis" : 1,
    "outbound_network_time_in_millis" : 1,
    "searches" : [
      {
        "query" : [
          {
            "type" : "ConstantScoreQuery",
            "description" : "ConstantScore(timestamp:[1674187200000 TO 1674273600000])",
            "time_in_nanos" : 21370639248,
            "breakdown" : {
              "set_min_competitive_score_count" : 0,
              "match_count" : 0,
              "shallow_advance_count" : 0,
              "set_min_competitive_score" : 0,
              "next_doc" : 21367383614,
              "match" : 0,
              "next_doc_count" : 270770395,
              "score_count" : 0,
              "compute_max_score_count" : 0,
              "compute_max_score" : 0,
              "advance" : 10310,
              "advance_count" : 34,
              "score" : 0,
              "build_scorer_count" : 68,
              "create_weight" : 64269,
              "shallow_advance" : 0,
              "create_weight_count" : 1,
              "build_scorer" : 3181055
            },
            "children" : [
              {
                "type" : "IndexOrDocValuesQuery",
                "description" : "timestamp:[1674187200000 TO 1674273600000]",
                "time_in_nanos" : 7104083495,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 7100917190,
                  "match" : 0,
                  "next_doc_count" : 270770395,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 6680,
                  "advance_count" : 34,
                  "score" : 0,
                  "build_scorer_count" : 68,
                  "create_weight" : 2390,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 3157235
                }
              }
            ]
          }
        ],
        "rewrite_time" : 41430,
        "collector" : [
          {
            "name" : "SimpleFieldCollector",
            "reason" : "search_top_hits",
            "time_in_nanos" : 10709644881
          }
        ]
      }
    ],
    "aggregations" : [ ]
  },
1 Like

Hi @reta
Sorry, I didn’t post the hit result. 90+% of hits come from the my-system-syslog-000015 index and other hits are from my-system-syslog-000013, my-system-syslog-000014 and my-system-syslog-000016
It seems to me that OpenSearch spent a long time on those indices because data matched the range is stored there? For example, in the posts above, the query on [g1LncdRoRCez4uKHcw12YQ][my-system-syslog-000014][1] took 20+ seconds, and the query on [g1LncdRoRCez4uKHcw12YQ][my-system-syslog-000022][1] took less than 0.001 second

If I change the time window to 2023-01-29T08:00~2023-01-30T08:00

GET /my-system-syslog/_search
{
  "profile": true,
  "version": true,
  "size": 1,
  "sort": [
    {
      "timestamp": {
        "order": "desc",
        "unmapped_type": "boolean"
      }
    }
  ],
  "stored_fields": [
    "*"
  ],
  "docvalue_fields": [
    {
      "field": "@timestamp",
      "format": "date_time"
    },
    {
      "field": "timestamp",
      "format": "date_time"
    }
  ],
  "_source": {
    "excludes": []
  },
  "query": {
    "bool": {
      "must": [],
      "filter": [
        {
          "match_all": {}
        },
        {
          "range": {
            "timestamp": {              
              "gte": "2023-01-29T08:00:00.000Z",
              "lte": "2023-01-30T08:00:00.000Z",
              "format": "strict_date_optional_time"
            }
          }
        }
      ],
      "should": [],
      "must_not": []
    }
  }
}

It took

  "took" : 20907,
  "timed_out" : false,
  "_shards" : {
    "total" : 93,
    "successful" : 93,
    "skipped" : 36,
    "failed" : 0
  },

And most hits come from my-system-syslog-000025, and OpenSearch spent 20+ seconds on those shards this time

{
        "id" : "[6brGb5iXQkuT9g-r-7gmLQ][my-system-syslog-000025][2]",
        "inbound_network_time_in_millis" : 4,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674979200000 TO 1675065600000])",
                "time_in_nanos" : 22544082638,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 22481607398,
                  "match" : 0,
                  "next_doc_count" : 287588758,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 11759,
                  "advance_count" : 33,
                  "score" : 0,
                  "build_scorer_count" : 66,
                  "create_weight" : 9360,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 62454121
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674979200000 TO 1675065600000]",
                    "time_in_nanos" : 7464949888,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 7402511307,
                      "match" : 0,
                      "next_doc_count" : 287588758,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 8019,
                      "advance_count" : 33,
                      "score" : 0,
                      "build_scorer_count" : 66,
                      "create_weight" : 610,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 62429952
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 13029,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 11194781857
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      
      
      {
        "id" : "[DD0Tc2xHS2qCuzDfQQ9njA][my-system-syslog-000025][1]",
        "inbound_network_time_in_millis" : 2,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674979200000 TO 1675065600000])",
                "time_in_nanos" : 22733486898,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 22715494900,
                  "match" : 0,
                  "next_doc_count" : 287621497,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 9990,
                  "advance_count" : 31,
                  "score" : 0,
                  "build_scorer_count" : 62,
                  "create_weight" : 14600,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 17967408
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674979200000 TO 1675065600000]",
                    "time_in_nanos" : 7683831834,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 7665875226,
                      "match" : 0,
                      "next_doc_count" : 287621497,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 4560,
                      "advance_count" : 31,
                      "score" : 0,
                      "build_scorer_count" : 62,
                      "create_weight" : 1860,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 17950188
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 20399,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 9494239033
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },
      
    {
        "id" : "[ftRbiVU1ScygA2Vg8Qctog][my-system-syslog-000025][0]",
        "inbound_network_time_in_millis" : 2,
        "outbound_network_time_in_millis" : 0,
        "searches" : [
          {
            "query" : [
              {
                "type" : "ConstantScoreQuery",
                "description" : "ConstantScore(timestamp:[1674979200000 TO 1675065600000])",
                "time_in_nanos" : 22495147351,
                "breakdown" : {
                  "set_min_competitive_score_count" : 0,
                  "match_count" : 0,
                  "shallow_advance_count" : 0,
                  "set_min_competitive_score" : 0,
                  "next_doc" : 22448914064,
                  "match" : 0,
                  "next_doc_count" : 287617313,
                  "score_count" : 0,
                  "compute_max_score_count" : 0,
                  "compute_max_score" : 0,
                  "advance" : 8719,
                  "advance_count" : 32,
                  "score" : 0,
                  "build_scorer_count" : 64,
                  "create_weight" : 16810,
                  "shallow_advance" : 0,
                  "create_weight_count" : 1,
                  "build_scorer" : 46207758
                },
                "children" : [
                  {
                    "type" : "IndexOrDocValuesQuery",
                    "description" : "timestamp:[1674979200000 TO 1675065600000]",
                    "time_in_nanos" : 7514239925,
                    "breakdown" : {
                      "set_min_competitive_score_count" : 0,
                      "match_count" : 0,
                      "shallow_advance_count" : 0,
                      "set_min_competitive_score" : 0,
                      "next_doc" : 7468050098,
                      "match" : 0,
                      "next_doc_count" : 287617313,
                      "score_count" : 0,
                      "compute_max_score_count" : 0,
                      "compute_max_score" : 0,
                      "advance" : 6589,
                      "advance_count" : 32,
                      "score" : 0,
                      "build_scorer_count" : 64,
                      "create_weight" : 1790,
                      "shallow_advance" : 0,
                      "create_weight_count" : 1,
                      "build_scorer" : 46181448
                    }
                  }
                ]
              }
            ],
            "rewrite_time" : 19600,
            "collector" : [
              {
                "name" : "SimpleFieldCollector",
                "reason" : "search_top_hits",
                "time_in_nanos" : 11167170260
              }
            ]
          }
        ],
        "aggregations" : [ ]
      },