Findings were not creating for valid documents

Plugins Security Analytics
1

In my cluster i have few detector and suddenly some detectors stopped working. when checking that i can see some errors in cluster level.

[2026-04-01T08:36:51,975][DEBUG][o.o.a.s.TransportSearchAction][1baf01041046fd159da64b23544b7d71] All shards failed for phase: [query]
org.opensearch.index.mapper.MapperParsingException: failed to parse
  at org.opensearch.index.mapper.DocumentParser.wrapInMapperParsingException(DocumentParser.java:206)
  at org.opensearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:99)
  at org.opensearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:253)
  at org.opensearch.percolator.PercolateQueryBuilderExt.doToQuery(PercolateQueryBuilderExt.java:525)
  at org.opensearch.index.query.AbstractQueryBuilder.toQuery(AbstractQueryBuilder.java:141)
  at org.opensearch.index.query.BoolQueryBuilder.addBooleanClauses(BoolQueryBuilder.java:348)
  at org.opensearch.index.query.BoolQueryBuilder.doToQuery(BoolQueryBuilder.java:331)
  at org.opensearch.index.query.AbstractQueryBuilder.toQuery(AbstractQueryBuilder.java:141)
  at org.opensearch.index.query.QueryShardContext.lambda$toQuery$3(QueryShardContext.java:575)
  at org.opensearch.index.query.QueryShardContext.toQuery(QueryShardContext.java:587)
  at org.opensearch.index.query.QueryShardContext.toQuery(QueryShardContext.java:574)
  at org.opensearch.search.SearchService.parseSource(SearchService.java:1592)
  at org.opensearch.search.SearchService.createContext(SearchService.java:1311)
  at org.opensearch.search.SearchService.executeQueryPhase(SearchService.java:877)
  at org.opensearch.search.SearchService$2.lambda$onResponse$0(SearchService.java:843)
  at org.opensearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:74)
  at org.opensearch.action.ActionRunnable$2.doRun(ActionRunnable.java:89)
  at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52)
  at org.opensearch.threadpool.TaskAwareRunnable.doRun(TaskAwareRunnable.java:78)
  at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52)
  at org.opensearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:59)
  at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:986)
  at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52)
  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
  at java.lang.Thread.run(Thread.java:1583)
Caused by: java.lang.IllegalArgumentException: Cannot write to a field alias [Whitelisted_registry_create_key_8m3ufZsBcmJk0YA8W1o0].
  at org.opensearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:772)
  at org.opensearch.index.mapper.DocumentParser.parseValue(DocumentParser.java:1143)
  at org.opensearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:571)
  at org.opensearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:521)
  at org.opensearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:141)
  at org.opensearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:96)
  ... 24 more

Here its mentioned as my detector uses some mapping which is not there in my index, but when checking my mapping i can see the correct mapping in my detector queries.

{
  ".opensearch-sap-xdr_windows-detectors-queries-optimized-d6fa3730-ab67-4881-91b0-bcdebd962b80-000001": {
    "mappings": {
      "_meta": {
        "schema_version": 1
      },
      "properties": {
        "EventId_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
          "type": "alias",
          "path": "eventName_registry_create_key_8m3ufZsBcmJk0YA8W1o0"
        },
        "EventType_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
          "type": "alias",
          "path": "log.attributes.EventType_registry_create_key_8m3ufZsBcmJk0YA8W1o0"
        },
        "Image_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
          "type": "alias",
          "path": "log.attributes.Image_registry_create_key_8m3ufZsBcmJk0YA8W1o0"
        },
        "TargetObject_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
          "type": "alias",
          "path": "log.attributes.TargetObject_registry_create_key_8m3ufZsBcmJk0YA8W1o0"
        },
        "Whitelisted_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
          "type": "alias",
          "path": "log.attributes.Whitelisted_registry_create_key_8m3ufZsBcmJk0YA8W1o0"
        },
        "droppedAttributesCount_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
          "type": "long"
        },
        "eventName_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
          "type": "integer"
        },
        "flags_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
          "type": "long"
        },
        "index": {
          "type": "text"
        },
        "instrumentationScope": {
          "properties": {
            "version_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
              "type": "keyword"
            }
          }
        },
        "log": {
          "properties": {
            "attributes": {
              "properties": {
                "EventType_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
                  "type": "keyword"
                },
                "Image_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
                  "type": "text",
                  "analyzer": "rule_analyzer"
                },
                "IngestedTimestamp_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
                  "type": "date"
                },
                "TargetObject_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
                  "type": "text",
                  "analyzer": "rule_analyzer"
                },
                "Whitelisted_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
                  "type": "boolean"
                }
              }
            }
          }
        },
        "monitor_id": {
          "type": "text"
        },
        "observedTimestamp_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
          "type": "date"
        },
        "query": {
          "type": "percolator_ext"
        },
        "resource": {
          "properties": {
            "attributes": {
              "properties": {
                "DeviceName_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
                  "type": "keyword"
                },
                "EndpointId_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
                  "type": "long"
                },
                "ExternalIp_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
                  "type": "keyword"
                },
                "LocalIp_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
                  "type": "keyword"
                },
                "TenantId_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
                  "type": "keyword"
                }
              }
            }
          }
        },
        "schemaUrl_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
          "type": "keyword"
        },
        "serviceName_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
          "type": "keyword"
        },
        "severityNumber_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
          "type": "integer"
        },
        "severityText_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
          "type": "keyword"
        },
        "spanId_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
          "type": "keyword"
        },
        "time_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
          "type": "date"
        },
        "traceId_registry_create_key_8m3ufZsBcmJk0YA8W1o0": {
          "type": "keyword"
        }
      }
    }
  }
}

Any solution that would help me identify the root cause will be more helpful. Looking forward the exact cause why I’m getting this error?

2

@Aravinth Thank you for the question, can you provide further details below:

  • What version of OS are you using? Did you upgrade recently?
  • Can you provide the mapping of the source index? Has it changed recently?
  • Was the field mapping step done automatically or manually?
  • Detector definition of working vs not-working detector.

This would provide better understanding of the issue you are seeing.

3

@Anthony

  1. I am using AWS opensearch. Previously the version was in 3.3 and recently we have updated the version to 3.5. (Updation happened on last week)
  2. Mappings of source Index
{
  "registry-create-key-000061": {
    "mappings": {
      "dynamic": "strict",
      "properties": {
        "EventId": {
          "type": "alias",
          "path": "eventName"
        },
        "EventType": {
          "type": "alias",
          "path": "log.attributes.EventType"
        },
        "Image": {
          "type": "alias",
          "path": "log.attributes.Image"
        },
        "TargetObject": {
          "type": "alias",
          "path": "log.attributes.TargetObject"
        },
        "Whitelisted": {
          "type": "alias",
          "path": "log.attributes.Whitelisted"
        },
        "droppedAttributesCount": {
          "type": "long"
        },
        "eventName": {
          "type": "integer"
        },
        "flags": {
          "type": "long"
        },
        "instrumentationScope": {
          "properties": {
            "version": {
              "type": "keyword"
            }
          }
        },
        "log": {
          "properties": {
            "attributes": {
              "properties": {
                "EventType": {
                  "type": "keyword"
                },
                "Image": {
                  "type": "text"
                },
                "IngestedTimestamp": {
                  "type": "date"
                },
                "TargetObject": {
                  "type": "text"
                },
                "Whitelisted": {
                  "type": "boolean"
                }
              }
            }
          }
        },
        "observedTimestamp": {
          "type": "date"
        },
        "resource": {
          "properties": {
            "attributes": {
              "properties": {
                "DeviceName": {
                  "type": "keyword"
                },
                "EndpointId": {
                  "type": "long"
                },
                "ExternalIp": {
                  "type": "keyword"
                },
                "LocalIp": {
                  "type": "keyword"
                },
                "TenantId": {
                  "type": "keyword"
                }
              }
            }
          }
        },
        "schemaUrl": {
          "type": "keyword"
        },
        "serviceName": {
          "type": "keyword"
        },
        "severityNumber": {
          "type": "integer"
        },
        "severityText": {
          "type": "keyword"
        },
        "spanId": {
          "type": "keyword"
        },
        "time": {
          "type": "date"
        },
        "traceId": {
          "type": "keyword"
        }
      }
    }
  }
}

Sample incoming event

{
  "took": 11,
  "timed_out": false,
  "_shards": {
    "total": 20,
    "successful": 20,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 10000,
      "relation": "gte"
    },
    "max_score": 1,
    "hits": [
      {
        "_index": "registry-create-key-000058",
        "_id": "9efe89f2ed5244d296e9559d95495788",
        "_score": 1,
        "_source": {
          "traceId": "25e7e230c208422bae70726f1d49b606",
          "spanId": "7c7f2c77ce2244e68d522bb8b08386d6",
          "severityText": "",
          "flags": 0,
          "time": "2026-03-27T05:35:13.146292200Z",
          "severityNumber": 0,
          "droppedAttributesCount": 0,
          "serviceName": null,
          "observedTimestamp": "1970-01-01T00:00:00Z",
          "schemaUrl": "",
          "eventName": 40000,
          "resource.attributes.ExternalIp": "125.21.99.218",
          "resource.attributes.EndpointId": "18398973186809856",
          "resource.attributes.LocalIp": "192.168.28.70",
          "resource.attributes.TenantId": "210279814660096556",
          "resource.attributes.DeviceName": "DESKTOP-3GQJ4OP",
          "instrumentationScope.version": "1.2.1.2",
          "log.attributes.Whitelisted": false,
          "log.attributes.TargetObject": """HKLM\SYSTEM\CONTROLSET001\SERVICES\TCPIP\PARAMETERS""",
          "log.attributes.Image": """C:\PROGRAM FILES\TIGHTVNC\TVNSERVER.EXE""",
          "log.attributes.EventType": "CREATEKEY",
          "log.attributes.IngestedTimestamp": "2026-03-27T05:35:18.700000000Z"
        }
      }
    ]
  }
}

No mappings has been changed in last one month

  1. Filed mapping are created using index templates. First I am creating an index template and using ISM policy I am rolling over the index.
{
  "index_patterns": [
    "registry-create-key-*"
  ],
  "template": {
    "settings": {
      "index.plugins.index_state_management.rollover_alias": "registry_create_key"
    },
    "mappings": {
      "dynamic": "strict",
      "properties": {
        "traceId": {
          "type": "keyword"
        },
        "Whitelisted": {
          "path": "log.attributes.Whitelisted",
          "type": "alias"
        },
        "instrumentationScope": {
          "type": "object",
          "properties": {
            "version": {
              "type": "keyword"
            }
          }
        },
        "log": {
          "type": "object",
          "properties": {
            "attributes": {
              "type": "object",
              "properties": {
                "Whitelisted": {
                  "type": "boolean"
                },
                "EventType": {
                  "type": "keyword"
                },
                "IngestedTimestamp": {
                  "type": "date"
                },
                "Image": {
                  "type": "text"
                },
                "TargetObject": {
                  "type": "text"
                }
              }
            }
          }
        },
        "EventType": {
          "path": "log.attributes.EventType",
          "type": "alias"
        },
        "resource": {
          "type": "object",
          "properties": {
            "attributes": {
              "type": "object",
              "properties": {
                "TenantId": {
                  "type": "keyword"
                },
                "ExternalIp": {
                  "type": "keyword"
                },
                "LocalIp": {
                  "type": "keyword"
                },
                "EndpointId": {
                  "type": "long"
                },
                "DeviceName": {
                  "type": "keyword"
                }
              }
            }
          }
        },
        "flags": {
          "type": "long"
        },
        "severityNumber": {
          "type": "integer"
        },
        "serviceName": {
          "type": "keyword"
        },
        "Image": {
          "path": "log.attributes.Image",
          "type": "alias"
        },
        "TargetObject": {
          "path": "log.attributes.TargetObject",
          "type": "alias"
        },
        "schemaUrl": {
          "type": "keyword"
        },
        "spanId": {
          "type": "keyword"
        },
        "severityText": {
          "type": "keyword"
        },
        "eventName": {
          "type": "integer"
        },
        "EventId": {
          "path": "eventName",
          "type": "alias"
        },
        "droppedAttributesCount": {
          "type": "long"
        },
        "time": {
          "type": "date"
        },
        "observedTimestamp": {
          "type": "date"
        }
      }
    },
    "aliases": {
      "security_data": {}
    }
  },
  "composed_of": [],
  "name": "registry-create-key",
  "priority": "undefined",
  "_meta": {
    "flow": "simple"
  }
}
  1. Events are ingested into the index without any error. I have created detectors pointing to that index and rules were added. Now the incoming documents are matching the rules in detectors but findings weren’t created. At time when checking that only i found out the above issues I shared in description.