How to get hits data in message

Plugins Alerting
1

Versions (relevant - OpenSearch/Dashboard/Server OS/Browser):
opensearch 2.16version

this is below is my hits in alert

{
    "_shards": {
        "total": 1,
        "failed": 0,
        "successful": 1,
        "skipped": 0
    },
    "hits": {
        "hits": [
            {
                "_index": "ss4o_metrics-otel-2024.08.29",
                "_source": {
                    "unit": "{cpu}",
                    "exemplars": [],
                    "kind": "GAUGE",
                    "name": "k8s.pod.cpu.usage",
                    "flags": 0,
                    "description": "Total CPU usage (sum of all cores per second) averaged over the sample window",
                    "startTime": "2024-08-29T05:29:48Z",
                    "attributes": {
                        "instrumentationScope.name": "otelcol/kubeletstatsreceiver",
                        "resource.attributes.k8s@pod@uid": "7a65a103-6cea-46d4-8363-c1c154a09e49",
                        "resource.attributes.k8s@namespace@name": "default",
                        "resource.attributes.k8s@node@name": "minikube",
                        "resource.attributes.k8s@deployment@name": "custommvc-open",
                        "resource.attributes.k8s@pod@start_time": "2024-08-29T05:29:35Z",
                        "resource.attributes.k8s@pod@name": "custommvc-open-85fd6ddd9-hp944",
                        "instrumentationScope.version": "0.106.1"
                    },
                    "time": "2024-08-29T08:40:30.643429481Z",
                    "serviceName": null,
                    "value": 0.540842557,
                    "schemaUrl": ""
                },
                "_id": "BaRLnZEBrIfLV-3HG99U",
                "_score": 9.211788
            }
        ],
        "total": {
            "value": 1,
            "relation": "eq"
        },
        "max_score": 9.211788
    },
    "took": 12,
    "timed_out": false
}

i want to get notification in attributes in resource.attributes.k8s@deployment@name and
resource.attributes.k8s@pod@name
i am try like this below but this not working pod name and deployment there are show empty in mail


{{#ctx.results.0.hits.hits}}
 - name: {{_source.name}}
 - cpu_usage: {{_source.value}}
 - pod: {{_source.attributes.resource.attributes.k8s@pod@name}} 
 - deployment: {{_source.attributes.resource.attributes.k8s@deployment@name}} 
{{/ctx.results.0.hits.hits}}
2

@naman Have you tried escaping @ sign or using _ instead of @ in the field name?

3

i can’t change @ sign to _

4

@naman What about escaping ‘@’ sign in the alert definition?

5

i am solved this problem to use data prepper to change data like below

    otel-metrics-pipeline:
      workers: 8      
      source:
        otel_metrics_source:
          health_check_service: true
          ssl: false     
      processor:       
        - otel_metrics_raw_processor:
            calculate_histogram_buckets: true
            calculate_exponential_histogram_buckets: true
            exponential_histogram_max_allowed_scale: 10
            flatten_attributes: false
       - rename_keys:                  
            
            - from_key: "attributes/resource.attributes.k8s@pod@name"
              to_key: "attributes/resource.attributes.kts_pod_name"
              overwrite_if_to_key_exists: true  
            - from_key: "attributes/resource.attributes.k8s@pod@uid"
              to_key: "attributes/resource.attributes.kts_pod_uid"
              overwrite_if_to_key_exists: true  
            - from_key: "attributes/resource.attributes.k8s@pod@start_time"
              to_key: "attributes/resource.attributes.kts_pod._starttime"
              overwrite_if_to_key_exists: true             
            
             
      sink:
        - opensearch:
            hosts: ["https://opensearch-cluster-master.myopensearch.svc.cluster.local:9200"]
            username: "admin"
            password: "TadhakDev01"
            insecure: true
            index_type: custom
            index: ss4o_metrics-otel-%{yyyy.MM.dd}
            bulk_size: 4

Output

{
    "_shards": {
        "total": 1,
        "failed": 0,
        "successful": 1,
        "skipped": 0
    },
    "hits": {
        "hits": [
            {
                "_index": "ss4o_metrics-otel-2024.08.29",
                "_source": {
                    "unit": "{cpu}",
                    "exemplars": [],
                    "kind": "GAUGE",
                    "name": "k8s.pod.cpu.usage",
                    "flags": 0,
                    "description": "Total CPU usage (sum of all cores per second) averaged over the sample window",
                    "startTime": "2024-08-29T05:29:48Z",
                    "attributes": {
                        "instrumentationScope.name": "otelcol/kubeletstatsreceiver",
                        "resource.attributes.kts_pod_uid": "7a65a103-6cea-46d4-8363-c1c154a09e49",   <-- change here                   
                        "resource.attributes.kts_pod_start_time": "2024-08-29T05:29:35Z", <-- change here
                        "resource.attributes.kts_pod_name": "custommvc-open-85fd6ddd9-hp944", <-- change here
                        "instrumentationScope.version": "0.106.1"
                    },
                    "time": "2024-08-29T08:40:30.643429481Z",
                    "serviceName": null,
                    "value": 0.540842557,
                    "schemaUrl": ""
                },
                "_id": "BaRLnZEBrIfLV-3HG99U",
                "_score": 9.211788
            }
        ],
        "total": {
            "value": 1,
            "relation": "eq"
        },
        "max_score": 9.211788
    },
    "took": 12,
    "timed_out": false
}