Versions (relevant - OpenSearch/Dashboard/Server OS/Browser):
1.3
Describe the issue:
We are migrating from ElasticSearch 7.10 to OpenSearch 1.3.
We are using low level REST client to perform Index Transform as High Level REST client of OpenSearch does not support it.
We get errors in pipeline aggregations while using the below Opensearch payload corresponding to also attached ElasticSearch payload to create Index Transform.
Are pipeline aggregations supported in Index transform APIs?
Illegal argument exception (for e1) is thrown in this config and also similarly in 2 other places
"socagtec2": {
"sum": {
"field": "ca1.value",
"script": {
"source": "if (doc['e1'].value.toInstant().toEpochMilli() < params['c2']) return 0; else return BigDecimal.valueOf(_value).setScale(2, RoundingMode.HALF_UP);",
"lang": "painless",
"params": {
"c2": epochtime1
}
}
}
Configuration:
ElasticSearch Tranform Index payload
{
"id": "tid",
"source": {
"index": [
"sourceIndex"
],
"query": {
"bool": {
"must": [
{
"range": {
"e1": {
"from": "fromDate",
"to": "toDate",
"include_lower": true,
"include_upper": true,
"boost": 1.0
}
}
},
{
"terms": {
"qualifiers.b1.keyword": [
"a",
"b"
],
"boost": 1.0
}
},
{
"term": {
"add_or_subtract.keyword": {
"value": "ADD",
"boost": 1.0
}
}
}
],
"must_not": [
{
"exists": {
"field": "r1",
"boost": 1.0
}
}
],
"adjust_pure_negative": true,
"boost": 1.0
}
}
},
"dest": {
"index": "targetIndex"
},
"frequency": "1h",
"sync": {
"time": {
"field": "@timestamp",
"delay": "1ms"
}
},
"pivot": {
"group_by": {
"qualifiers-aaid1": {
"terms": {
"field": "qualifiers.aid1.keyword"
}
},
"qualifiers-ac1": {
"terms": {
"field": "qualifiers.ac1.keyword"
}
},
"qualifiers-c1": {
"terms": {
"field": "qualifiers.c1.keyword"
}
}
},
"aggregations": {
"by_b1_id": {
"terms": {
"field": "qualifiers.b1.keyword",
"size": 10,
"min_doc_count": 1,
"shard_min_doc_count": 0,
"show_term_doc_count_error": false,
"order": [
{
"_count": "desc"
},
{
"_key": "asc"
}
]
},
"aggregations": {
"by_date": {
"date_histogram": {
"field": "e1",
"format": "yyyy-MM-dd",
"fixed_interval": "1d",
"offset": 0,
"order": {
"_key": "asc"
},
"keyed": false,
"min_doc_count": 0,
"extended_bounds": {
"min": "mindate",
"max": "maxdate"
}
},
"aggregations": {
"soa": {
"sum": {
"field": "a1",
"script": {
"source": "BigDecimal.valueOf(_value).setScale(2, RoundingMode.HALF_UP)",
"lang": "painless"
}
}
},
"soagtec1": {
"sum": {
"field": "a1",
"script": {
"source": "if (doc['e1'].value.toInstant().toEpochMilli() < params['c1']) return 0; else return BigDecimal.valueOf(_value).setScale(2, RoundingMode.HALF_UP)",
"lang": "painless",
"params": {
"c1": epochtime1
}
}
}
},
"soca": {
"sum": {
"field": "ca1.value",
"script": {
"source": "BigDecimal.valueOf(_value).setScale(2, RoundingMode.HALF_UP)",
"lang": "painless"
}
}
},
"socagtec1": {
"sum": {
"field": "ca1.value",
"script": {
"source": "if (doc['e1'].value.toInstant().toEpochMilli() < params['c1']) return 0; else return BigDecimal.valueOf(_value).setScale(2, RoundingMode.HALF_UP)",
"lang": "painless",
"params": {
"c1": epochtime1
}
}
}
},
"dcgtec1": {
"sum": {
"script": {
"source": "if (doc['e1'].value.toInstant().toEpochMilli() < params['c1']) return 0; else return 1;",
"lang": "painless",
"params": {
"c1": epochtime1
}
}
}
},
"coa": {
"value_count": {
"field": "a1"
}
},
"coca": {
"value_count": {
"field": "ca1.value"
}
},
"mtlt": {
"max": {
"field": "@timestamp"
}
},
"csa": {
"cumulative_sum": {
"buckets_path": [
"soagtec1"
]
}
},
"csca": {
"cumulative_sum": {
"buckets_path": [
"socagtec1"
]
}
},
"dc": {
"bucket_script": {
"buckets_path": {
"count": "_count"
},
"script": {
"source": "params.count",
"lang": "painless"
},
"gap_policy": "skip"
}
},
"cdc": {
"cumulative_sum": {
"buckets_path": [
"dcgtec1"
]
}
}
}
}
}
}
}
},
"description": "desc",
"settings": {
"max_page_search_size": 10000
}
}
Opensearch Transform Index payload
{
"transform": {
"enabled": true,
"continuous": true,
"schedule": {
"interval": {
"period": 60,
"unit": "minutes",
"start_time": epochtime1
}
},
"description": "",
"metadata_id": "metadataid",
"source_index": "sourceIndex",
"target_index": "targetIndex",
"data_selection_query": {
"bool": {
"must": [
{
"range": {
"e1": {
"include_lower": true,
"include_upper": true,
"boost": 1,
"from": "fromDate",
"to": "toDate"
}
}
},
{
"terms": {
"boost": 1,
"qualifiers.b1.keyword": [
"a",
"b"
]
}
},
{
"term": {
"add_or_subtract.keyword": {
"value": "ADD",
"boost": 1
}
}
}
],
"must_not": [
{
"exists": {
"field": "r1",
"boost": 1
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
},
"page_size": 1000,
"groups": [
{
"date_histogram": {
"source_field": "e1",
"calendar_interval": "day"
}
},
{
"terms": {
"source_field": "qualifiers.b1.keyword"
}
},
{
"terms": {
"source_field": "qualifiers.c1.keyword"
}
},
{
"terms": {
"source_field": "qualifiers.ac1.keyword"
}
},
{
"terms": {
"source_field": "qualifiers.aid1.keyword"
}
}
],
"aggregations": {
"soa": {
"sum": {
"field": "a1",
"script": {
"source": "BigDecimal.valueOf(_value).setScale(2, RoundingMode.HALF_UP);",
"lang": "painless"
}
}
},
"soagtec": {
"sum": {
"field": "a1",
"script": {
"source": "if (doc['e1'].value.toInstant().toEpochMilli() < params['c2']) return 0; else return BigDecimal.valueOf(_value).setScale(2, RoundingMode.HALF_UP);",
"lang": "painless",
"params": {
"c2": epochtime1
}
}
}
},
"soca": {
"sum": {
"field": "ca1.value",
"script": {
"source": "BigDecimal.valueOf(_value).setScale(2, RoundingMode.HALF_UP);",
"lang": "painless"
}
}
},
"socagtec2": {
"sum": {
"field": "ca1.value",
"script": {
"source": "if (doc['e1'].value.toInstant().toEpochMilli() < params['c2']) return 0; else return BigDecimal.valueOf(_value).setScale(2, RoundingMode.HALF_UP);",
"lang": "painless",
"params": {
"c2": epochtime1
}
}
}
},
"dc": {
"bucket_script": {
"buckets_path": {
"count": "_count"
},
"script": {
"source": "params.count",
"lang": "painless"
},
"gap_policy": "skip"
}
},
"dcgtec2": {
"sum": {
"script": {
"source": "if (doc['e1'].value.toInstant().toEpochMilli() < params['c2']) return 0; else return 1;",
"lang": "painless",
"params": {
"c2": epochtime1
}
}
}
},
"coa": {
"value_count": {
"field": "a1"
}
},
"coca": {
"value_count": {
"field": "ca1.value"
}
},
"mtlt": {
"max": {
"field": "@timestamp"
}
}
}
}
}
Relevant Logs or Screenshots: