Unable to sort query results with "search_pipeline" hybrid query

Versions:
OpenSearch version 2.13.0, Dashboard 2.13.0

When performing a hybrid query with a search_pipeline, the sort tag does not work as expected.

Reproduction:

Create index.

PUT /knn-sample-index
{
  "settings": {
    "index":{
       "knn":"true"
    }
  },
  "mappings": {
    "properties": {
      "textVector": {
        "type": "knn_vector",
        "dimension": 5,
        "method": {
          "engine": "faiss",
          "space_type": "innerproduct",
          "name": "hnsw",
          "parameters": {
            "ef_construction": 1024,
            "m": 64
          }
        }
      },
      "imageVector": {
        "type": "knn_vector",
        "dimension": 5,
        "method": {
          "engine": "faiss",
          "space_type": "innerproduct",
          "name": "hnsw",
          "parameters": {
            "ef_construction": 1024,
            "m": 64
          }
        }
      },
      "name": {
        "type": "text"
      },
      "timestamp": {
        "type": "date"
      }
    }
  }
}

Add data.

PUT /knn-sample-index/_doc/1
{
"name": "Apple iPhone 13, 128GB, Pink - Unlocked (Renewed)",
"imageVector": [-0.5548,  0.3177,  0.4558, -0.5047,  0.3590],
"textVector": [-0.5313,  0.5175,  0.1438, -0.5471, -0.3605],
"timestamp": "2023-06-02T16:31:15.4488190Z"
}

PUT /knn-sample-index/_doc/2
{
"name": "ASUS Chromebook Plus CX34 Laptop, 14 Display (1920x1080), Intel® Core i3-1215U Processor, 8GB RAM, 256GB UFS Storage, ChromeOS, White, CX3402CBA-DH386-WH",
"imageVector": [0.4884, 0.3328, 0.4026, 0.5094, 0.4786],
"textVector": [0.4003, 0.4308, 0.4791, 0.3798, 0.5295],
"timestamp": "2024-08-10T16:31:15.4488190Z"
}

PUT /knn-sample-index/_doc/3
{
"name": "Sony 50 Inch 4K Ultra HD TV X85K Series: LED Smart Google TV with Dolby Vision HDR and Native 120HZ Refresh Rate KD50X85K- Latest Model, Black",
"imageVector": [0.4387, 0.4179, 0.4221, 0.5298, 0.4173],
"timestamp": "2018-03-24T16:31:15.4488190Z"
}

PUT /knn-sample-index/_doc/4
{
"name": "Amazon Kindle Paperwhite (16 GB) – Now with a larger display, adjustable warm light, increased battery life, and faster page turns – Without Lockscreen Ads – Black",
"textVector": [0.4794, 0.4412, 0.4031, 0.4694, 0.4390],
"timestamp": "2019-01-21T16:31:15.4488190Z"
}

PUT /knn-sample-index/_doc/5
{
"name": "SAMSUNG Galaxy S24+ Plus Cell Phone, 256GB AI Smartphone, Unlocked Android, 50MP Camera, Fastest Processor, Long Battery Life, US Version 2024 Cobalt Violet",
"timestamp": "2020-12-05T16:31:15.4488190Z"
}

Query the index (note the temporary search pipeline).

GET knn-sample-index/_search
{
  "query": {
    "hybrid": {
      "queries": [
        {
          "match": {
            "name": {
              "query": "Apple iPhone 13"
            }
          }
        },
        {
          "knn": {
            "imageVector": {
              "vector": [1,1,1,1,1],
              "k": 3
            }
          }
        }
      ]
    }
  },
  "sort": [
    {
      "timestamp": {
        "order": "asc"
      }
    }
  ],
  "search_pipeline" : {
    "phase_results_processors": [
      {
        "normalization-processor": {
          "normalization": {
            "technique": "min_max"
          }
          , "combination": {
              "technique": "arithmetic_mean"
            , "parameters": {
              "weights": [0.5, 0.5]
            }
          }
          , "ignore_failure": false
        }
      }
    ]
  }
}

Query results.
Note that the results are not sorted by timestamp. I also tried sorting by the score with no success.

{
  "took": 2,
  "timed_out": false,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 3,
      "relation": "eq"
    },
    "max_score": 0.5005,
    "hits": [
      {
        "_index": "knn-sample-index",
        "_id": "1",
        "_score": 0.5005,
        "_source": {
          "name": "Apple iPhone 13, 128GB, Pink - Unlocked (Renewed)",
          "imageVector": [
            -0.5548,
            0.3177,
            0.4558,
            -0.5047,
            0.359
          ],
          "textVector": [
            -0.5313,
            0.5175,
            0.1438,
            -0.5471,
            -0.3605
          ],
          "timestamp": "2023-06-02T16:31:15.4488190Z"
        }
      },
      {
        "_index": "knn-sample-index",
        "_id": "3",
        "_score": 0.5,
        "_source": {
          "name": "Sony 50 Inch 4K Ultra HD TV X85K Series: LED Smart Google TV with Dolby Vision HDR and Native 120HZ Refresh Rate KD50X85K- Latest Model, Black",
          "imageVector": [
            0.4387,
            0.4179,
            0.4221,
            0.5298,
            0.4173
          ],
          "timestamp": "2018-03-24T16:31:15.4488190Z"
        }
      },
      {
        "_index": "knn-sample-index",
        "_id": "2",
        "_score": 0.49674845,
        "_source": {
          "name": "ASUS Chromebook Plus CX34 Laptop, 14 Display (1920x1080), Intel® Core i3-1215U Processor, 8GB RAM, 256GB UFS Storage, ChromeOS, White, CX3402CBA-DH386-WH",
          "imageVector": [
            0.4884,
            0.3328,
            0.4026,
            0.5094,
            0.4786
          ],
          "textVector": [
            0.4003,
            0.4308,
            0.4791,
            0.3798,
            0.5295
          ],
          "timestamp": "2024-08-10T16:31:15.4488190Z"
        }
      }
    ]
  }
}

If I remove the search pipeline, the sort works as expected.

GET knn-sample-index/_search
{
  "query": {
    "script_score": {
      "query": {
        "hybrid": {
          "queries": [
            {
              "match": {
                "name": {
                  "query": "Apple iPhone 13"
                }
              }
            },
            {
              "knn": {
                "imageVector": {
                  "vector": [1,1,1,1,1],
                  "k": 3
                }
              }
            }
          ]
        }
      },
      "script": {
        "source": "_score"
      }
    }
  },
  "sort": [
    {
      "timestamp": {
        "order": "desc"
      }
    }
  ]
}
{
  "took": 2,
  "timed_out": false,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 3,
      "relation": "eq"
    },
    "max_score": null,
    "hits": [
      {
        "_index": "knn-sample-index",
        "_id": "2",
        "_score": null,
        "_source": {
          "name": "ASUS Chromebook Plus CX34 Laptop, 14 Display (1920x1080), Intel® Core i3-1215U Processor, 8GB RAM, 256GB UFS Storage, ChromeOS, White, CX3402CBA-DH386-WH",
          "imageVector": [
            0.4884,
            0.3328,
            0.4026,
            0.5094,
            0.4786
          ],
          "textVector": [
            0.4003,
            0.4308,
            0.4791,
            0.3798,
            0.5295
          ],
          "timestamp": "2024-08-10T16:31:15.4488190Z"
        },
        "sort": [
          1723307475448
        ]
      },
      {
        "_index": "knn-sample-index",
        "_id": "1",
        "_score": null,
        "_source": {
          "name": "Apple iPhone 13, 128GB, Pink - Unlocked (Renewed)",
          "imageVector": [
            -0.5548,
            0.3177,
            0.4558,
            -0.5047,
            0.359
          ],
          "textVector": [
            -0.5313,
            0.5175,
            0.1438,
            -0.5471,
            -0.3605
          ],
          "timestamp": "2023-06-02T16:31:15.4488190Z"
        },
        "sort": [
          1685723475448
        ]
      },
      {
        "_index": "knn-sample-index",
        "_id": "3",
        "_score": null,
        "_source": {
          "name": "Sony 50 Inch 4K Ultra HD TV X85K Series: LED Smart Google TV with Dolby Vision HDR and Native 120HZ Refresh Rate KD50X85K- Latest Model, Black",
          "imageVector": [
            0.4387,
            0.4179,
            0.4221,
            0.5298,
            0.4173
          ],
          "timestamp": "2018-03-24T16:31:15.4488190Z"
        },
        "sort": [
          1521909075448
        ]
      }
    ]
  }
}

This topic was automatically closed 60 days after the last reply. New replies are no longer allowed.