Versions:
OpenSearch version 2.13.0, Dashboard 2.13.0
When performing a hybrid query with a search_pipeline
, the sort
tag does not work as expected.
Reproduction:
Create index.
PUT /knn-sample-index
{
"settings": {
"index":{
"knn":"true"
}
},
"mappings": {
"properties": {
"textVector": {
"type": "knn_vector",
"dimension": 5,
"method": {
"engine": "faiss",
"space_type": "innerproduct",
"name": "hnsw",
"parameters": {
"ef_construction": 1024,
"m": 64
}
}
},
"imageVector": {
"type": "knn_vector",
"dimension": 5,
"method": {
"engine": "faiss",
"space_type": "innerproduct",
"name": "hnsw",
"parameters": {
"ef_construction": 1024,
"m": 64
}
}
},
"name": {
"type": "text"
},
"timestamp": {
"type": "date"
}
}
}
}
Add data.
PUT /knn-sample-index/_doc/1
{
"name": "Apple iPhone 13, 128GB, Pink - Unlocked (Renewed)",
"imageVector": [-0.5548, 0.3177, 0.4558, -0.5047, 0.3590],
"textVector": [-0.5313, 0.5175, 0.1438, -0.5471, -0.3605],
"timestamp": "2023-06-02T16:31:15.4488190Z"
}
PUT /knn-sample-index/_doc/2
{
"name": "ASUS Chromebook Plus CX34 Laptop, 14 Display (1920x1080), Intel® Core i3-1215U Processor, 8GB RAM, 256GB UFS Storage, ChromeOS, White, CX3402CBA-DH386-WH",
"imageVector": [0.4884, 0.3328, 0.4026, 0.5094, 0.4786],
"textVector": [0.4003, 0.4308, 0.4791, 0.3798, 0.5295],
"timestamp": "2024-08-10T16:31:15.4488190Z"
}
PUT /knn-sample-index/_doc/3
{
"name": "Sony 50 Inch 4K Ultra HD TV X85K Series: LED Smart Google TV with Dolby Vision HDR and Native 120HZ Refresh Rate KD50X85K- Latest Model, Black",
"imageVector": [0.4387, 0.4179, 0.4221, 0.5298, 0.4173],
"timestamp": "2018-03-24T16:31:15.4488190Z"
}
PUT /knn-sample-index/_doc/4
{
"name": "Amazon Kindle Paperwhite (16 GB) – Now with a larger display, adjustable warm light, increased battery life, and faster page turns – Without Lockscreen Ads – Black",
"textVector": [0.4794, 0.4412, 0.4031, 0.4694, 0.4390],
"timestamp": "2019-01-21T16:31:15.4488190Z"
}
PUT /knn-sample-index/_doc/5
{
"name": "SAMSUNG Galaxy S24+ Plus Cell Phone, 256GB AI Smartphone, Unlocked Android, 50MP Camera, Fastest Processor, Long Battery Life, US Version 2024 Cobalt Violet",
"timestamp": "2020-12-05T16:31:15.4488190Z"
}
Query the index (note the temporary search pipeline).
GET knn-sample-index/_search
{
"query": {
"hybrid": {
"queries": [
{
"match": {
"name": {
"query": "Apple iPhone 13"
}
}
},
{
"knn": {
"imageVector": {
"vector": [1,1,1,1,1],
"k": 3
}
}
}
]
}
},
"sort": [
{
"timestamp": {
"order": "asc"
}
}
],
"search_pipeline" : {
"phase_results_processors": [
{
"normalization-processor": {
"normalization": {
"technique": "min_max"
}
, "combination": {
"technique": "arithmetic_mean"
, "parameters": {
"weights": [0.5, 0.5]
}
}
, "ignore_failure": false
}
}
]
}
}
Query results.
Note that the results are not sorted by timestamp. I also tried sorting by the score with no success.
{
"took": 2,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 3,
"relation": "eq"
},
"max_score": 0.5005,
"hits": [
{
"_index": "knn-sample-index",
"_id": "1",
"_score": 0.5005,
"_source": {
"name": "Apple iPhone 13, 128GB, Pink - Unlocked (Renewed)",
"imageVector": [
-0.5548,
0.3177,
0.4558,
-0.5047,
0.359
],
"textVector": [
-0.5313,
0.5175,
0.1438,
-0.5471,
-0.3605
],
"timestamp": "2023-06-02T16:31:15.4488190Z"
}
},
{
"_index": "knn-sample-index",
"_id": "3",
"_score": 0.5,
"_source": {
"name": "Sony 50 Inch 4K Ultra HD TV X85K Series: LED Smart Google TV with Dolby Vision HDR and Native 120HZ Refresh Rate KD50X85K- Latest Model, Black",
"imageVector": [
0.4387,
0.4179,
0.4221,
0.5298,
0.4173
],
"timestamp": "2018-03-24T16:31:15.4488190Z"
}
},
{
"_index": "knn-sample-index",
"_id": "2",
"_score": 0.49674845,
"_source": {
"name": "ASUS Chromebook Plus CX34 Laptop, 14 Display (1920x1080), Intel® Core i3-1215U Processor, 8GB RAM, 256GB UFS Storage, ChromeOS, White, CX3402CBA-DH386-WH",
"imageVector": [
0.4884,
0.3328,
0.4026,
0.5094,
0.4786
],
"textVector": [
0.4003,
0.4308,
0.4791,
0.3798,
0.5295
],
"timestamp": "2024-08-10T16:31:15.4488190Z"
}
}
]
}
}
If I remove the search pipeline
, the sort
works as expected.
GET knn-sample-index/_search
{
"query": {
"script_score": {
"query": {
"hybrid": {
"queries": [
{
"match": {
"name": {
"query": "Apple iPhone 13"
}
}
},
{
"knn": {
"imageVector": {
"vector": [1,1,1,1,1],
"k": 3
}
}
}
]
}
},
"script": {
"source": "_score"
}
}
},
"sort": [
{
"timestamp": {
"order": "desc"
}
}
]
}
{
"took": 2,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 3,
"relation": "eq"
},
"max_score": null,
"hits": [
{
"_index": "knn-sample-index",
"_id": "2",
"_score": null,
"_source": {
"name": "ASUS Chromebook Plus CX34 Laptop, 14 Display (1920x1080), Intel® Core i3-1215U Processor, 8GB RAM, 256GB UFS Storage, ChromeOS, White, CX3402CBA-DH386-WH",
"imageVector": [
0.4884,
0.3328,
0.4026,
0.5094,
0.4786
],
"textVector": [
0.4003,
0.4308,
0.4791,
0.3798,
0.5295
],
"timestamp": "2024-08-10T16:31:15.4488190Z"
},
"sort": [
1723307475448
]
},
{
"_index": "knn-sample-index",
"_id": "1",
"_score": null,
"_source": {
"name": "Apple iPhone 13, 128GB, Pink - Unlocked (Renewed)",
"imageVector": [
-0.5548,
0.3177,
0.4558,
-0.5047,
0.359
],
"textVector": [
-0.5313,
0.5175,
0.1438,
-0.5471,
-0.3605
],
"timestamp": "2023-06-02T16:31:15.4488190Z"
},
"sort": [
1685723475448
]
},
{
"_index": "knn-sample-index",
"_id": "3",
"_score": null,
"_source": {
"name": "Sony 50 Inch 4K Ultra HD TV X85K Series: LED Smart Google TV with Dolby Vision HDR and Native 120HZ Refresh Rate KD50X85K- Latest Model, Black",
"imageVector": [
0.4387,
0.4179,
0.4221,
0.5298,
0.4173
],
"timestamp": "2018-03-24T16:31:15.4488190Z"
},
"sort": [
1521909075448
]
}
]
}
}