Update/upsert API not generating embeddings

OpenSearch
, ,
1

Versions (relevant - OpenSearch/Dashboard/Server OS/Browser):
OpenSearch 3.2

Describe the issue:
When Update or upsert api is used through a pipeline with text embedding processor.
The embeddings are not generated. Below is sample examples

Configuration:
Ingest-pipeline sample:

POST /_ingest/pipeline/ingest-sample
{
“description”: “A text embedding pipeline”,
“processors”: [
{
“text_embedding”: {
“model_id”: “GjOB3poBn6Lh5apOZkru”,
“field_map”: {
“passage_text”: “passage_embedding”,
“passage_text_esp”:“passage_embedding_esp”
}
}
}
]
}

sample index mappping:
PUT sample_upsert
{
“settings”: {
“index.knn”: true
},
“mappings”: {
“properties”: {
“id”: {
“type”: “text”
},
“passage_embedding”: {
“type”: “knn_vector”,
“dimension”: 384,
“method”: {
“engine”: “lucene”,
“space_type”: “l2”,
“name”: “hnsw”,
“parameters”: {}
}
},
“passage_text”: {
“type”: “text”
},
“passage_embedding_esp”: {
“type”: “knn_vector”,
“dimension”: 384,
“method”: {
“engine”: “lucene”,
“space_type”: “l2”,
“name”: “hnsw”,
“parameters”: {}
}
},
“passage_text_esp”: {
“type”: “text”
}
}
}
}

bulk indexing:

POST _bulk?pipeline=ingest-sample
{ “index”: { “_index”: “sample_upsert”, “_id”: “1” } }
{ “passage_text”: “OpenSearch is a distributed search and analytics engine built on Apache Lucene.”}
{ “index”: { “_index”: “sample_upsert”, “_id”: “2” } }
{ “id”: “2”,“passage_text”: “Vector search enables semantic similarity queries using embeddings.”}
{ “index”: { “_index”: “sample_upsert”, “_id”: “3” } }
{ “id”: “3”,“passage_text”: “HNSW is an efficient algorithm for approximate nearest neighbor search.”}
{ “index”: { “_index”: “sample_upsert”, “_id”: “4” } }
{ “id”: “4”,“passage_text”: “Lucene provides the underlying engine for indexing and search in OpenSearch.”,“passage_text_esp”:“Tejas asasdfsaf”}
{ “index”: { “_index”: “sample_upsert”, “_id”: “5” } }
{ “id”: “5”,“passage_text”: “KNN vector fields require fixed-dimension numeric arrays.”}
{ “index”: { “_index”: “sample_upsert”, “_id”: “6” } }
{ “id”: “6”,“passage_text”: “asfasf vector fields require fixed-dimension numeric arrays.”,“passage_text_esp”: “asfasf vector fields require fixed-dimension numeric arrays.”}

Results after indexing:
{
“took”: 6,
“timed_out”: false,
“_shards”: {
“total”: 1,
“successful”: 1,
“skipped”: 0,
“failed”: 0
},
“hits”: {
“total”: {
“value”: 6,
“relation”: “eq”
},
“max_score”: 1,
“hits”: [
{
“_index”: “sample_upsert”,
“_id”: “1”,
“_score”: 1,
“_source”: {
“passage_embedding”: [-0.007618205, -0.12449546, -0.07068711, 0.023918627, 0.03183581, 0.006794856, -0.088174626, 0.0795943, 0.003379296, -0.00456478, -0.040207747, -0.036876637, -0.0060705296, -0.05039652, -0.042221963, 0.036289386, 0.065989226, 0.048648328, -0.0029988186, -0.043093998, 0.04451932, 0.016119953, 0.028996836, -0.09868701, -0.038604967, 0.021652801, -0.04431272, -0.01525925, 0.010797342, 0.026381781, 0.047766645, 0.013257394, 0.07410067, 0.063976236, -0.0054817707, -0.013008901, 0.025627593, -0.018411508, -0.07775482, 0.0044963625, -0.012974912, -0.022081949, -0.009969791, 0.07348842, -0.06679005, 0.017094437, 0.01401719, -0.015782593, -0.05626308, -0.10896774, -0.12584978, 0.012776297, -0.033284653, 0.00405426, 0.011665775, 0.012498158, 0.05931097, -0.046826378, -0.05874136, -0.08510736, -0.0063988916, 0.045039345, -0.032737765, 0.038130485, -0.005690992, -0.0054599787, 0.021667752, 0.021571718, 0.084668346, -0.11729825, 0.0069885943, 0.00003861718, 0.0072435355, 0.057419326, -0.05609006, -0.015880529, 0.05289239, -0.0122999605, 0.017133923, -0.025902152, -0.03495367, 0.029739255, -0.021258285, -0.027329873, 0.0207369, -0.053263627, 0.008751338, -0.03401203, 0.106724, 0.08036759, 0.03395531, -0.0933592, -0.017384512, -0.07505579, -0.05057079, 0.060487133, -0.0096535375, 0.070372306, 0.051851038, 0.056161348, 0.02443373, 0.054225538, -0.02509025, -0.086175725, 0.0052897274, 0.039240085, 0.010637186, 0.001652668, 0.100477874, -0.0106907515, -0.01837536, 0.0056404485, -0.011620829, -0.038633853, -0.0020648036, -0.057951815, -0.05073396, -0.017121105, -0.016841441, 0.07512155, -0.048602074, 0.033891723, -0.03012159, -0.059499465, 0.0033789447, 0.023561044, -0.045274004, 0.0146817425, 0.05083395, -0.06487579, -0.053615868, -0.048190355, -0.0725443, -0.015916565, 0.0063755554, 0.068007864, -0.14396267, -0.03637558, -0.029574085, 0.12870549, -0.030006915, 0.007517172, -0.040454056, 0.029278073, -0.013857964, 0.057034854, -0.007852395, -0.08811524, -0.038467366, -0.118184455, 0.006460906, -0.021444678, -0.005329451, 0.055575646, 0.12385583, -0.027921308, 0.015567428, 0.01142302, 0.061276797, -0.0025944465, -0.056644768, 0.031000143, 0.034764644, 0.030389424, -0.050967753, 0.01721623, 0.04164685, -0.06571451, -0.030882992, 0.039673775, 0.17951207, -0.040763028, -0.06799738, -0.046716217, -0.09070291, 0.012840421, 0.0012591316, -0.10047082, 0.041023616, -0.024911674, 0.07412509, 0.056156263, 0.0059032906, 0.072423376, -0.0053593772, 0.0450322, -0.020883324, 0.062483285, 0.0004747986, 0.021178575, 0.090924665, 0.022623682, 0.118021235, -0.034156952, -0.0074527487, -0.035716087, -0.026211752, 0.008382586, -0.022515034, 0.018315472, 0.052413806, 0.063706905, -0.009189424, 0.08313663, -0.042466868, 0.045560684, 0.036871884, -0.0874193, 0.0875801, -0.03275983, -0.0072930795, 0.05113917, -0.042755865, 0.032768507, 0.052567743, -0.10525277, -0.02719073, -0.029630374, -0.06798056, 0.003969431, -0.0977614, -0.03144359, -0.044548623, 4.2930583e-34, -0.021955233, -0.07938259, -0.01117786, 0.11587874, 0.008032554, -0.009475844, -0.035477962, -0.048920702, 0.017668797, 0.0559207, -0.0008926655, -0.014045459, 0.10551985, 0.014927949, -0.041460115, -0.009259608, 0.1168599, -0.046686083, 0.019883592, 0.084991254, 0.042133983, 0.06966021, -0.08915175, 0.04893512, -0.008834569, 0.029766832, 0.050601386, -0.05447974, -0.051683016, 0.0039793127, -0.059146684, -0.046170387, -0.013281222, -0.0056915157, 0.055509683, 0.017775942, 0.06316756, -0.033150677, -0.027371319, -0.02215056, 0.09020225, 0.029263496, -0.035145566, 0.034195017, 0.01690272, 0.050560284, -0.052250747, 0.10246292, 0.008951967, -0.006402803, 0.042136915, 0.06740218, 0.049400702, -0.06144623, -0.0030806293, 0.035457946, -0.09610013, 0.019675052, -0.050974965, 0.053372305, -0.002122789, 0.021090351, -0.07727128, 0.111024566, -0.0029783861, -0.016643098, 0.03325896, 0.053723924, -0.017440086, -0.048171367, 0.016001668, -0.08023724, 0.069875136, -0.007468593, -0.025369868, -0.0093642995, 0.06344899, 0.07054538, -0.0063873585, 0.0768059, 0.022020949, 0.042925823, 0.06342566, -0.00094325235, 0.030751148, -0.01597421, 0.05126634, 0.058427375, -0.06226805, -0.04533932, 0.0050785905, -0.008301512, -0.024726508, -0.015786504, 0.020728283, 2.2564186e-32, -0.085180156, 0.031743836, -0.017703166, 0.0037338687, 0.050241064, -0.06294875, -0.024826212, 0.08359072, 0.020416008, -0.0638867, 0.03309067, -0.016229626, -0.09689802, -0.039812185, 0.005321282, -0.020728724, -0.026837789, 0.030925639, -0.07597506, -0.024597107, -0.04103661, 0.040700603, 0.04677524, -0.00042441566, 0.027651506, -0.029930841, -0.025837827, -0.041494496, -0.042300306, -0.05729614, 0.003644277, 0.04490548, -0.025618097, -0.019266006, 0.08151229, 0.066591166, -0.036309008, 0.029868005, -0.051396765, 0.07756222, 0.05705612, 0.05662109, 0.044994656, 0.026981538, -0.04469518, -0.01693661, 0.024606152, -0.025954999, 0.06726945, -0.018137477, -0.043776654, -0.073129535, 0.048786886, -0.055552334, 0.009150494, 0.019953124, -0.025348177, -0.1486717, 0.045520574, 0.03205424, 0.01934653, -0.06819554, 0.036262326, -0.03984884],
“passage_text”: “OpenSearch is a distributed search and analytics engine built on Apache Lucene.”
}
},
{
“_index”: “sample_upsert”,
“_id”: “2”,
“_score”: 1,
“_source”: {
“passage_text”: “Vector search enables semantic similarity queries using embeddings.”,
“passage_embedding”: [0.026083095, -0.067727886, 0.043178726, -0.031542167, -0.004399777, -0.013967995, -0.060490288, 0.09509109, 0.00430261, -0.00627726, 0.080520645, -0.052798033, -0.049123146, -0.026538057, 0.014245002, -0.011685951, 0.0075974283, 0.13315636, -0.031493794, -0.05012921, 0.018730676, 0.030582182, 0.020599771, -0.01380502, -0.038501535, 0.016096147, -0.006807534, 0.013631381, 0.054272264, -0.0018038357, 0.12999037, 0.040789552, 0.06143593, 0.06944648, -0.057859898, -0.012377196, 0.001080881, 0.07147598, -0.04261844, -0.044861127, 0.027512576, 0.00792085, 0.030904537, 0.036124576, -0.025240015, 0.0715859, 0.031187076, -0.0062395963, 0.038677376, -0.019052638, 0.03661084, -0.043285843, -0.054633494, 0.04230883, 0.07051349, 0.049957406, 0.052673765, -0.06387623, -0.025792643, -0.12222278, -0.0063197757, 0.016614728, 0.017301125, 0.028382314, 0.025326028, -0.045545314, 0.042833623, 0.005541816, -0.024577096, 0.00968863, 0.028421259, 0.03309431, -0.078234784, 0.124440715, -0.11707664, 0.031267982, 0.05710486, -0.07583989, 0.035170246, -0.022910044, 0.011208475, 0.060569104, -0.009157271, -0.016050877, -0.03798157, -0.074460626, 0.10066296, -0.054388784, 0.06916367, -0.016868904, -0.033817463, -0.013092841, 0.026025236, -0.044133443, 0.026729885, -0.025777854, -0.013786166, 0.037714083, 0.061615895, 0.04515479, 0.030900449, 0.026327644, 0.0111779, 0.046724442, -0.022795724, 0.0006107711, -0.01332171, -0.058014993, 0.017698556, -0.055159174, 0.030442184, 0.08886656, -0.04125317, -0.08136491, 0.07030764, -0.05128093, -0.007745783, 0.12516777, 0.020515973, 0.045411535, -0.004825414, -0.0024048677, -0.051119573, 0.041593004, 0.038086798, -0.121287495, 0.012435926, 0.011650843, 0.02966544, 0.0059807114, 0.057140768, 0.06796318, -0.042273443, 0.070861585, -0.03928181, 0.119387485, -0.051519666, -0.11299718, -0.06892981, 0.06976501, -0.039368853, 0.0021159514, 0.058385145, 0.060598005, -0.098474056, 0.05302948, -0.16474278, -0.09827998, 0.031845506, -0.05129408, 0.014515105, -0.0406978, 0.02522653, 0.073249206, 0.042583324, 0.020570504, -0.006650351, -0.0072417734, -0.014669841, -0.047966473, -0.005201732, 0.053364657, -0.0025263058, -0.039364304, 0.0073737013, -0.017037245, -0.0941305, -0.07798143, -0.027609278, -0.028755449, 0.09940158, -0.028812764, -0.041964065, 0.060993083, -0.016161593, -0.023302216, -0.019840317, -0.057562355, 0.008787507, 0.008831341, 0.012702984, -0.02656768, 0.040744945, 0.03427152, -0.011669889, -0.009199524, 0.07497578, -0.028168956, -0.04926027, -0.035611432, 0.06821179, -0.065339506, 0.068973385, 0.05512504, 0.029516798, -0.019814305, 0.0833075, 0.01890894, 0.06179381, -0.016343502, -0.045608208, -0.11647018, -0.048096754, -0.03637659, -0.0331258, -0.026581155, 0.008210577, 0.02652634, -0.022560894, -0.08315273, -0.071950436, 0.013933145, -0.067799725, -0.029684471, 0.022390928, -0.113514, -0.026566021, -0.05366493, 0.027222464, 0.03472746, -0.017937606, -0.06020265, 0.029669115, -4.726142e-33, -0.079081506, -0.05615268, -0.011220467, 0.0074068503, -0.020694034, 0.031213587, 0.032043815, -0.0287803, -0.058529504, -0.05151516, -0.05462611, 0.027346218, -0.007450919, -0.009736384, -0.03331555, 0.09083842, -0.0024224175, 0.011424607, 0.022567078, 0.05948949, -0.014895368, -0.006669302, -0.07416872, 0.028840369, 0.0013264145, 0.026844021, 0.09908265, -0.04473576, 0.009974985, -0.027619805, -0.018952908, -0.07316988, -0.015699103, -0.008837521, -0.069700055, 0.02761229, 0.10022452, -0.008285672, -0.036324646, -0.070111185, 0.043701272, -0.016710376, -0.04223605, 0.0023628965, -0.11396539, 0.035824753, -0.009629503, 0.026583914, 0.07202662, 0.023823703, 0.0651506, 0.06373037, -0.06894256, -0.057089515, -0.07004105, 0.00563206, -0.07796538, 0.044103842, 0.00004574336, -0.042121634, -0.08946181, -0.034714896, -0.019962609, 0.053306773, -0.019965192, -0.02403357, 0.02275521, 0.059889223, -0.0137637695, -0.117569804, -0.0059109363, -0.10713167, 0.039872482, 0.01665636, -0.024587963, 0.061060756, 0.039661977, 0.039273452, -0.012802754, 0.07109534, -0.010605673, -0.012310071, -0.0039405106, -0.000557138, -0.006689474, 0.054849874, 0.033014473, 0.10975497, -0.020810908, -0.028282266, 0.021713153, -0.0015410905, 0.0006583933, 0.055106726, 0.026486602, 1.988666e-33, 0.0560179, 0.10378463, -0.03797907, -0.013366286, 0.03214722, -0.059459526, 0.02710635, 0.07157912, -0.04176391, 0.10804123, 0.02460433, 0.05546473, 0.011393352, -0.02736592, 0.038296975, -0.0067399833, 0.024409782, 0.0065370444, -0.012152865, -0.09282115, -0.027151264, 0.05831001, -0.014300941, 0.019107426, -0.01875739, 0.101735465, -0.06254901, 0.07654346, 0.1335275, 0.01948249, -0.0021897673, 0.026711788, -0.051988628, -0.03422898, 0.046197765, -0.013204699, -0.0085312035, -0.03628559, -0.059543137, -0.016776837, 0.0994159, -0.042206842, 0.042302284, -0.09695073, 0.073436245, 0.10136779, -0.013382648, -0.045513164, 0.031353094, -0.061408926, -0.004448006, -0.00274579, -0.000257434, 0.06641312, -0.02766623, -0.040209055, -0.07851905, 0.04211129, 0.002630921, 0.01265773, -0.0049386118, -0.00023492052, -0.0076999147, -0.07195595],
“id”: “2”
}
},
{
“_index”: “sample_upsert”,
“_id”: “3”,
“_score”: 1,
“_source”: {
“passage_text”: “HNSW is an efficient algorithm for approximate nearest neighbor search.”,
“passage_embedding”: [-0.031280912, -0.021837853, -0.03606383, -0.0046238247, -0.042972583, -0.024706919, -0.10526674, 0.044436317, -0.030777764, -0.05786313, 0.051826347, -0.012003693, 0.030389607, -0.0036672396, -0.09034428, -0.014327567, 0.0806074, 0.0002324767, 0.015883923, -0.14784048, 0.00925654, 0.019230181, 0.03222666, -0.043239348, 0.011278297, -0.011285865, 0.0063554263, 0.049749956, 0.028450536, -0.026043188, -0.0003512103, -0.007591146, 0.109154455, 0.0302321, 0.02446077, -0.017402867, 0.018546317, 0.09651604, -0.061056618, -0.0540702, 0.07059685, 0.039987754, 0.04688827, 0.05911543, -0.051750965, -0.018370455, -0.047234926, -0.023273954, -0.004610981, 0.0022577846, 0.006066136, -0.009905122, -0.040484898, 0.0290689, 0.06988019, 0.043878715, 0.0005521404, -0.061819416, -0.058434036, -0.008099028, 0.04146309, 0.019662537, 0.0068121064, -0.005285454, -0.0008661957, -0.03346035, 0.045839403, -0.05373032, -0.024831573, -0.04829399, -0.04532542, 0.04234834, -0.037092797, 0.027055418, -0.06269758, 0.010848952, -0.0049020015, -0.014243868, -0.030263418, -0.091097414, -0.033311337, 0.01954596, -0.050282665, -0.034263875, 0.077314764, 0.047010884, -0.00456487, -0.0354012, 0.07925111, -0.042334434, -0.050628044, -0.033866458, -0.105076104, 0.00586654, 0.0038743215, -0.01690701, 0.018871259, -0.0021629857, 0.05476943, 0.07976453, 0.05887606, 0.040958352, -0.013923776, -0.06787056, -0.013065776, -0.062256828, -0.048097484, 0.023165619, 0.06464759, -0.04738115, 0.02775369, -0.043858625, -0.012800379, 0.041825578, 0.06589009, -0.044615902, 0.08737557, 0.069544554, -0.08589572, 0.07971055, -0.06891913, -0.04458738, 0.02521024, -0.0034466875, 0.0025240253, -0.052188884, -0.07663519, -0.014361838, 0.06281471, 0.054270234, -0.013425431, -0.049311776, 0.051591568, 0.057998233, 0.023271574, 0.09308205, -0.04002608, -0.04514283, -0.10205475, 0.038408384, -0.00089701696, -0.026375182, 0.04671804, 0.037611984, 0.018142166, 0.073482394, -0.111161046, 0.007248173, 0.037945744, -0.120229155, -0.015114234, -0.08661489, -0.023969708, -0.022804243, 0.037547264, -0.05175553, 0.15743689, -0.0003381696, 0.030819941, 0.008354755, -0.015421928, 0.043515727, 0.053408917, 0.0032169293, -0.0922137, -0.08614422, -0.01794461, -0.118756644, -0.031207444, 0.019474804, 0.042110726, 0.018274372, 0.058540106, 0.056271616, -0.06071079, 0.02125822, -0.04212959, 0.01226756, 0.032558635, 0.008826348, -0.060385153, 0.026714195, -0.0064162924, 0.056378596, 0.073074095, -0.016196575, -0.006307942, 0.059676267, -0.024922466, -0.075763024, 0.020499751, -0.044113338, 0.07920474, -0.0098446915, 0.05579448, -0.027706578, 0.03928554, 0.0088133, 0.084644385, 0.03814679, -0.012600623, -0.02366114, -0.029198099, -0.03225692, 0.06998979, -0.029296083, -0.012231433, -0.0013439527, -0.058489352, 0.09496724, -0.06115307, 0.022030704, -0.05626612, -0.029824818, 0.036069028, -0.10266569, -0.058305573, -0.06514265, -0.02845902, 0.041554317, 0.047081854, -0.107312545, -0.078984, -1.5653426e-33, 0.0011035022, -0.06337935, 0.02070122, 0.016476927, -0.043388855, -0.009009131, 0.055458732, 0.048200525, 0.0025091472, 0.005384421, -0.017629497, 0.013951921, -0.0074014366, -0.03172225, 0.059083935, 0.06527458, 0.090112634, 0.007747049, 0.014850664, 0.05623441, 0.053751726, -0.03872071, -0.029093474, 0.10337587, 0.03877624, 0.023289578, -0.015021682, 0.018343614, -0.004036561, -0.07294433, -0.032020416, -0.047318656, 0.03167712, 0.023589129, 0.04040298, 0.0579965, 0.11382802, 0.077982664, -0.02833139, -0.028248819, 0.053342097, 0.05817646, -0.06101532, 0.03575331, -0.010055215, 0.013078544, -0.061622508, 0.030966789, -0.040057383, 0.09692365, -0.011184209, -0.036472544, -0.004391354, 0.030226354, -0.008617827, 0.093770675, -0.1761346, 0.008140162, 0.027566247, 0.008621067, -0.05517049, 0.0153903095, -0.040419437, 0.16534357, -0.10862348, -0.020114679, -0.0004009065, 0.06434986, -0.008973875, -0.015443258, -0.070903175, 0.020885183, 0.075139605, 0.029348673, -0.07494036, -0.04378943, 0.029583406, -0.051717192, -0.08188878, 0.051093616, -0.014616845, 0.046515513, 0.106287934, 0.01965578, 0.03936825, 0.086851195, 0.01272296, -0.0044801477, 0.015091581, -0.02171599, 0.003110283, -0.022294527, 0.00698221, -0.033730745, -0.04241854, 9.3347784e-33, -0.034942336, -0.020650916, -0.00201936, 0.030177774, -0.012302349, -0.0026954603, -0.027613806, 0.09711294, -0.036873147, -0.047229633, 0.038516097, 0.0030543988, -0.049866248, -0.013223888, -0.016120363, 0.023503276, 0.0413037, -0.015639655, -0.01753973, 0.06282044, -0.052751057, 0.05688037, 0.019272901, 0.039159056, 0.020768851, 0.041014258, -0.11390305, 0.008736953, 0.03211949, 0.00072152045, 0.022566974, 0.019941023, -0.020732645, 0.029785924, 0.015583456, 0.12095534, 0.0028472263, 0.005018528, -0.07730673, 0.060770947, 0.051949393, 0.045963347, 0.046184175, 0.03115966, 0.083770424, 0.018317893, 0.030671949, -0.09375895, 0.07373331, -0.11288941, -0.060277358, -0.0045368467, 0.029207999, -0.022283534, -0.020229563, -0.025989335, -0.025466524, -0.15212508, -0.0142035615, 0.007878304, -0.006480968, -0.04494819, -0.014092292, -0.037573427],
“id”: “3”
}
},
{
“_index”: “sample_upsert”,
“_id”: “5”,
“_score”: 1,
“_source”: {
“passage_text”: “KNN vector fields require fixed-dimension numeric arrays.”,
“passage_embedding”: [-0.020612774, 0.03967514, -0.056382306, 0.015943605, -0.008036526, -0.060305584, -0.09315985, -0.021453148, -0.008112472, -0.023438323, -0.030097485, -0.0037987744, -0.018696798, -0.025805427, -0.0034577274, 0.0037728797, -0.13716474, 0.045440543, 0.054476272, 0.09980573, -0.086042106, 0.03353749, -0.022885373, 0.032643806, -0.023203565, 0.025163723, -0.0093581155, 0.0016941683, -0.017617691, -0.06787213, -0.03180846, 0.0956388, 0.0017977884, 0.08736098, -0.07337206, -0.018704733, -0.007824896, 0.082441136, -0.00546949, 0.040648825, -0.022604812, -0.025821399, 0.036591563, 0.009482875, 0.02687369, 0.0792755, 0.0035747134, -0.021039832, 0.025995968, -0.020006517, -0.034344364, -0.017863128, -0.09875957, 0.03406955, 0.057077084, -0.0038826, -0.005179767, 0.030687947, 0.014304745, -0.105690055, -0.05600037, -0.04731618, 0.05447381, -0.0431799, -0.05210796, 0.019776909, 0.05285394, 0.014306385, -0.019750694, -0.009987338, -0.038386054, 0.061917376, -0.014392416, 0.029640183, 0.03150279, -0.020972678, -0.046424538, -0.044144437, 0.060152497, -0.013814643, -0.015360954, 0.05980114, 0.05535733, -0.04407122, 0.07544152, 0.080902375, -0.03809856, 0.010581814, 0.018424198, 0.027146457, 0.012757017, -0.019913446, 0.015187934, 0.027747573, 0.06659556, -0.017793432, -0.009425911, -0.0022865175, -0.012327983, 0.098162055, 0.089899816, -0.02017018, -0.0154127125, 0.06252777, -0.0077661825, -0.054230686, 0.06875253, -0.0033140802, 0.02196098, -0.00738153, 0.034372345, 0.093751505, -0.03731554, -0.10840965, 0.017995404, 0.092695534, -0.03606003, 0.044593822, 0.0222908, -0.04193709, -0.03075976, 0.01659414, -0.052999724, 0.026205461, 0.041411117, -0.056875385, -0.026846083, -0.0009477537, -0.014239625, 0.07414616, 0.05573658, -0.012133354, 0.030355414, 0.056772165, 0.006594309, 0.03244494, 0.058344007, 0.0029071446, -0.061988905, 0.02115022, -0.08263771, 0.010633303, 0.08391474, -0.008740618, 0.042341176, 0.08360001, -0.017424408, 0.034016825, 0.06573754, 0.025363693, -0.015506588, -0.038122915, -0.055396378, -0.045267574, -0.123761885, 0.019440457, 0.042739466, -0.006548206, -0.09204768, 0.03489529, 0.07037114, -0.04678789, 0.066529125, 0.006805137, -0.06983049, -0.08048979, 0.025784465, 0.0095156655, 0.042734824, -0.10606002, 0.02706176, -0.05072763, 0.062250987, 0.08032537, 0.06090524, 0.056029383, 0.010353017, -0.002776837, -0.04191407, 0.015255684, -0.088808104, 0.0382895, 0.086948834, 0.042871714, -0.05381428, -0.028125187, -0.012985826, 0.029310022, -0.11321682, -0.02521013, -0.0016317082, -0.009999566, -0.066732675, -0.016182581, 0.003360519, -0.030873818, 0.0061442996, -0.00079392927, 0.051053133, 0.109538935, -0.07065936, -0.013227411, -0.027632311, 0.027238071, 0.03235375, -0.015556257, -0.0254684, -0.008867507, -0.024546482, 0.015741074, -0.08348235, 0.054625917, 0.021311466, -0.038822815, -0.10530566, 0.023122435, -0.00042177032, -0.03613721, -0.048859112, 0.04688102, -0.036097173, -0.13317321, -0.039438095, -8.204497e-33, -0.08601132, -0.08725577, -0.076219514, 0.10561937, -0.035567366, 0.04110521, 0.02263005, -0.07729408, 0.020251568, -0.041778076, -0.033329017, -0.030595317, -0.07597107, 0.014182835, 0.045680445, -0.037202235, -0.10606898, -0.023199111, 0.039520953, 0.08244616, -0.042013627, 0.032639105, -0.013011338, 0.093790025, 0.06934033, 0.047717415, -0.02732434, -0.009609959, -0.056500535, 0.039955553, -0.04315015, -0.029359499, -0.016756436, 0.0074102776, 0.003535798, -0.015348298, 0.13425574, -0.035437103, -0.08381199, 0.07219648, 0.08608335, -0.041391432, -0.021290032, -0.029134644, -0.030716212, -0.051930793, 0.049906127, 0.029927157, 0.029029312, -0.04089377, -0.03509973, -0.009137969, 0.02437957, 0.012854805, -0.054299008, -0.05718741, -0.023531018, 0.07736854, 0.055333085, -0.10727971, -0.06195495, -0.095004216, 0.011387783, 0.062817685, 0.03607777, -0.088078566, -0.013745236, 0.052402318, -0.10214135, -0.09494098, 0.06392129, 0.016282754, 0.013597956, 0.055445522, -0.06837263, -0.014653982, -0.0010601395, 0.0042144544, 0.036198284, 0.021117838, -0.006564547, -0.029765574, 0.07686238, -0.013613692, 0.047875185, 0.10025504, 0.05914091, -0.027320914, -0.019605607, 0.010080223, 0.0945006, 0.0091896765, 0.010471584, 0.074330844, -0.02053418, 3.100102e-32, -0.02289784, 0.021989973, -0.10428657, -0.040256098, 0.040889848, -0.075252, 0.04906638, 0.047303226, -0.040175717, 0.04507337, 0.06777697, -0.067795396, -0.01267694, -0.03353565, 0.0073299464, -0.007448682, -0.053301074, 0.050983947, -0.031045675, 0.030126829, 0.028417358, 0.108609036, -0.12248448, 0.013105498, -0.002072105, -0.023679588, 0.035487443, -0.00017925231, -0.009551826, 0.102528624, -0.009902908, -0.0140382815, 0.049923904, 0.014617891, 0.00976243, 0.07819212, -0.051911123, -0.014992597, -0.07361949, -0.06668495, -0.025403477, -0.041736465, 0.04751572, -0.054365374, 0.05822107, 0.06817607, -0.021227313, -0.035161376, 0.056635644, 0.053816568, -0.0052261623, 0.04268152, 0.029059721, 0.02787956, 0.000047406116, -0.014063655, -0.041554477, -0.06820327, 0.03957388, 0.01514117, -0.033244867, -0.13600172, 0.014034533, -0.07026571],
“id”: “5”
}
},
{
“_index”: “sample_upsert”,
“_id”: “6”,
“_score”: 1,
“_source”: {
“passage_text”: “asfasf vector fields require fixed-dimension numeric arrays.”,
“passage_text_esp”: “asfasf vector fields require fixed-dimension numeric arrays.”,
“passage_embedding”: [0.01072704, 0.025694532, -0.031382076, 0.024874512, -0.026987137, -0.01032866, -0.032767095, -0.020898359, -0.045888733, -0.017222403, -0.012408084, -0.016562141, -0.0255221, 0.04830497, 0.07293453, 0.04949145, -0.09706565, 0.019075776, 0.09290193, 0.1168248, 0.021751152, 0.025458343, -0.010851581, 0.0297728, -0.060021643, 0.041545577, -0.044981103, -0.042624764, 0.03497176, -0.073487185, 0.005051537, 0.08872106, 0.0011634614, -0.016415084, -0.059983596, -0.030659067, 0.03200815, 0.04174323, 0.02716319, -0.013509032, -0.039272618, -0.020487007, 0.0015263101, 0.028726123, 0.028138602, 0.012854895, 0.009867581, 0.045861475, 0.037648827, -0.053435303, -0.02408884, -0.022777237, -0.03276013, 0.05119094, -0.007975884, -0.052767035, -0.04196187, -0.050760113, -0.03707873, -0.104220256, -0.10293207, -0.06006823, 0.009908563, -0.043020915, -0.0786695, 0.01069723, 0.042304337, -0.0039798827, 0.04858118, 0.033303596, -0.06904379, 0.033797666, 0.018885164, 0.020865707, 0.066735454, -0.0032580276, -0.024197837, -0.0037036317, 0.109328166, -0.028371358, -0.008890216, 0.026460063, 0.0061550546, -0.022848366, 0.044631146, -0.008196057, -0.023681883, -0.070899084, 0.01712555, -0.015085945, -0.0019761773, 0.0143613275, 0.043935966, 0.044224717, 0.08552811, -0.028682638, -0.03892936, -0.022544015, 0.012521425, 0.072604634, 0.047519747, 0.06624855, -0.035033993, 0.12545143, -0.074789315, -0.031351008, 0.032665342, 0.006668618, -0.029951746, -0.011046909, 0.062542364, 0.08949743, -0.03296978, -0.090013176, -0.026475925, 0.0655281, -0.06728133, 0.07985499, 0.04446329, -0.0718569, -0.018486183, -0.020138206, -0.036040053, -0.009210408, 0.06742051, -0.030435242, -0.062283125, -0.0001689811, -0.02146834, 0.10751599, 0.05649023, -0.090439126, 0.004295736, 0.0428772, 0.013859607, 0.05121965, 0.050865043, 0.02291196, -0.03733964, 0.011618358, -0.027290247, 0.071908176, 0.090382, 0.013616398, 0.0706777, 0.12023287, 0.010823674, 0.024341088, 0.05215431, 0.034290716, 0.012547685, -0.030099116, 0.0009039679, -0.021360967, -0.053667035, 0.087738186, -0.040047057, 0.025910815, -0.10712064, 0.019369463, 0.011581795, -0.00022565959, 0.09539108, 0.011769488, -0.04602777, -0.11132968, -0.019640448, 0.010725809, 0.00013805726, 0.002483253, 0.0148411235, -0.08831672, -0.0061035054, 0.0519261, 0.039127853, -0.019563451, 0.012324389, -0.048235904, -0.019465968, 0.0053983815, -0.08861492, 0.029568715, 0.07978889, 0.02072087, -0.07509665, -0.018244598, 0.03208166, 0.03097369, -0.121794835, -0.0015435322, -0.019788936, -0.025119137, -0.037122816, -0.010915106, 0.0125328535, -0.034826603, 0.011645789, 0.027423859, 0.060988262, 0.10230735, -0.066846184, 0.028399331, -0.055646583, 0.0072903503, 0.012678312, -0.04592136, -0.034156103, -0.04767544, 0.010022345, 0.03655502, -0.037961327, 0.019212665, 0.08177811, -0.05246402, -0.057526853, 0.0306948, -0.015069367, -0.06789071, -0.06824382, 0.034296677, 0.025175625, -0.1222606, -0.012267048, -6.3692156e-33, -0.06695666, -0.09256796, -0.027520576, 0.015875982, -0.07094035, -0.0031960066, 0.05485537, -0.07942127, 0.073692106, -0.04061019, -0.009018988, -0.03207307, -0.09980132, -0.022269385, -0.010783475, -0.031529687, -0.06747191, -0.088192545, 0.06023605, 0.047910057, 0.0031298276, 0.041095745, 0.04506125, 0.11660352, 0.058554094, 0.027683219, -0.034131825, -0.0075370236, -0.081955954, 0.053669967, -0.05548665, 0.0037342128, -0.012654296, 0.00588884, -0.011921794, -0.07399039, 0.03783106, -0.015327744, -0.05014004, 0.019823262, 0.042284634, 0.028254114, -0.016855055, -0.021764776, -0.105475746, -0.035133224, 0.040827278, 0.035146315, -0.023372542, -0.049775295, -0.09653039, -0.060373604, 0.012200464, 0.030828167, -0.04330903, 0.013243493, 0.034344167, 0.06107481, 0.04096722, -0.035803866, -0.022462651, -0.1032584, 0.05248223, -0.0029593124, 0.064318225, -0.10414462, 0.034810677, 0.0042904485, -0.0651975, -0.12164864, 0.009291307, -0.041769076, -0.05054591, 0.07354605, -0.09833419, -0.0026128285, 0.0070581725, -0.043213364, -0.017308844, 0.0801311, -0.0067197154, -0.06661673, -0.014538731, 0.04732792, 0.062657304, 0.020233292, 0.018428765, 0.05123823, 0.024488704, -0.025233662, 0.028782574, 0.0460028, 0.08635802, 0.0548346, 0.016579337, 3.7496462e-32, -0.018717604, 0.053009056, -0.09060177, -0.038905293, 0.08116858, -0.039393943, -0.013462924, -0.047281083, -0.013138307, -0.025784671, 0.03954526, 0.033510983, -0.025524782, -0.044231307, 0.013651797, -0.00627965, 0.00039079905, 0.091725774, -0.0014480667, -0.0076112375, 0.007970357, 0.091861546, -0.11491589, 0.012853278, -0.06528476, -0.019269263, 0.023365028, -0.004054526, 0.09008407, 0.18046612, -0.088137515, 0.014932047, 0.060067944, 0.032179266, -0.05143082, 0.08554405, 0.034904703, -0.06325008, -0.032084342, -0.0043214215, 0.010953515, -0.041213896, -0.024142234, -0.058931872, 0.048679736, 0.055450503, -0.07602062, -0.014719088, 0.067907244, 0.072052695, 0.07338123, 0.0370209, -0.007485006, 0.008103286, -0.00826996, 0.012282976, -0.04791335, -0.011939104, -0.0019211866, 0.031963453, 0.032881346, -0.12699214, 0.052015174, -0.05223969],
“passage_embedding_esp”: [0.01072704, 0.025694532, -0.031382076, 0.024874512, -0.026987137, -0.01032866, -0.032767095, -0.020898359, -0.045888733, -0.017222403, -0.012408084, -0.016562141, -0.0255221, 0.04830497, 0.07293453, 0.04949145, -0.09706565, 0.019075776, 0.09290193, 0.1168248, 0.021751152, 0.025458343, -0.010851581, 0.0297728, -0.060021643, 0.041545577, -0.044981103, -0.042624764, 0.03497176, -0.073487185, 0.005051537, 0.08872106, 0.0011634614, -0.016415084, -0.059983596, -0.030659067, 0.03200815, 0.04174323, 0.02716319, -0.013509032, -0.039272618, -0.020487007, 0.0015263101, 0.028726123, 0.028138602, 0.012854895, 0.009867581, 0.045861475, 0.037648827, -0.053435303, -0.02408884, -0.022777237, -0.03276013, 0.05119094, -0.007975884, -0.052767035, -0.04196187, -0.050760113, -0.03707873, -0.104220256, -0.10293207, -0.06006823, 0.009908563, -0.043020915, -0.0786695, 0.01069723, 0.042304337, -0.0039798827, 0.04858118, 0.033303596, -0.06904379, 0.033797666, 0.018885164, 0.020865707, 0.066735454, -0.0032580276, -0.024197837, -0.0037036317, 0.109328166, -0.028371358, -0.008890216, 0.026460063, 0.0061550546, -0.022848366, 0.044631146, -0.008196057, -0.023681883, -0.070899084, 0.01712555, -0.015085945, -0.0019761773, 0.0143613275, 0.043935966, 0.044224717, 0.08552811, -0.028682638, -0.03892936, -0.022544015, 0.012521425, 0.072604634, 0.047519747, 0.06624855, -0.035033993, 0.12545143, -0.074789315, -0.031351008, 0.032665342, 0.006668618, -0.029951746, -0.011046909, 0.062542364, 0.08949743, -0.03296978, -0.090013176, -0.026475925, 0.0655281, -0.06728133, 0.07985499, 0.04446329, -0.0718569, -0.018486183, -0.020138206, -0.036040053, -0.009210408, 0.06742051, -0.030435242, -0.062283125, -0.0001689811, -0.02146834, 0.10751599, 0.05649023, -0.090439126, 0.004295736, 0.0428772, 0.013859607, 0.05121965, 0.050865043, 0.02291196, -0.03733964, 0.011618358, -0.027290247, 0.071908176, 0.090382, 0.013616398, 0.0706777, 0.12023287, 0.010823674, 0.024341088, 0.05215431, 0.034290716, 0.012547685, -0.030099116, 0.0009039679, -0.021360967, -0.053667035, 0.087738186, -0.040047057, 0.025910815, -0.10712064, 0.019369463, 0.011581795, -0.00022565959, 0.09539108, 0.011769488, -0.04602777, -0.11132968, -0.019640448, 0.010725809, 0.00013805726, 0.002483253, 0.0148411235, -0.08831672, -0.0061035054, 0.0519261, 0.039127853, -0.019563451, 0.012324389, -0.048235904, -0.019465968, 0.0053983815, -0.08861492, 0.029568715, 0.07978889, 0.02072087, -0.07509665, -0.018244598, 0.03208166, 0.03097369, -0.121794835, -0.0015435322, -0.019788936, -0.025119137, -0.037122816, -0.010915106, 0.0125328535, -0.034826603, 0.011645789, 0.027423859, 0.060988262, 0.10230735, -0.066846184, 0.028399331, -0.055646583, 0.0072903503, 0.012678312, -0.04592136, -0.034156103, -0.04767544, 0.010022345, 0.03655502, -0.037961327, 0.019212665, 0.08177811, -0.05246402, -0.057526853, 0.0306948, -0.015069367, -0.06789071, -0.06824382, 0.034296677, 0.025175625, -0.1222606, -0.012267048, -6.3692156e-33, -0.06695666, -0.09256796, -0.027520576, 0.015875982, -0.07094035, -0.0031960066, 0.05485537, -0.07942127, 0.073692106, -0.04061019, -0.009018988, -0.03207307, -0.09980132, -0.022269385, -0.010783475, -0.031529687, -0.06747191, -0.088192545, 0.06023605, 0.047910057, 0.0031298276, 0.041095745, 0.04506125, 0.11660352, 0.058554094, 0.027683219, -0.034131825, -0.0075370236, -0.081955954, 0.053669967, -0.05548665, 0.0037342128, -0.012654296, 0.00588884, -0.011921794, -0.07399039, 0.03783106, -0.015327744, -0.05014004, 0.019823262, 0.042284634, 0.028254114, -0.016855055, -0.021764776, -0.105475746, -0.035133224, 0.040827278, 0.035146315, -0.023372542, -0.049775295, -0.09653039, -0.060373604, 0.012200464, 0.030828167, -0.04330903, 0.013243493, 0.034344167, 0.06107481, 0.04096722, -0.035803866, -0.022462651, -0.1032584, 0.05248223, -0.0029593124, 0.064318225, -0.10414462, 0.034810677, 0.0042904485, -0.0651975, -0.12164864, 0.009291307, -0.041769076, -0.05054591, 0.07354605, -0.09833419, -0.0026128285, 0.0070581725, -0.043213364, -0.017308844, 0.0801311, -0.0067197154, -0.06661673, -0.014538731, 0.04732792, 0.062657304, 0.020233292, 0.018428765, 0.05123823, 0.024488704, -0.025233662, 0.028782574, 0.0460028, 0.08635802, 0.0548346, 0.016579337, 3.7496462e-32, -0.018717604, 0.053009056, -0.09060177, -0.038905293, 0.08116858, -0.039393943, -0.013462924, -0.047281083, -0.013138307, -0.025784671, 0.03954526, 0.033510983, -0.025524782, -0.044231307, 0.013651797, -0.00627965, 0.00039079905, 0.091725774, -0.0014480667, -0.0076112375, 0.007970357, 0.091861546, -0.11491589, 0.012853278, -0.06528476, -0.019269263, 0.023365028, -0.004054526, 0.09008407, 0.18046612, -0.088137515, 0.014932047, 0.060067944, 0.032179266, -0.05143082, 0.08554405, 0.034904703, -0.06325008, -0.032084342, -0.0043214215, 0.010953515, -0.041213896, -0.024142234, -0.058931872, 0.048679736, 0.055450503, -0.07602062, -0.014719088, 0.067907244, 0.072052695, 0.07338123, 0.0370209, -0.007485006, 0.008103286, -0.00826996, 0.012282976, -0.04791335, -0.011939104, -0.0019211866, 0.031963453, 0.032881346, -0.12699214, 0.052015174, -0.05223969],
“id”: “6”
}
}
]
}
}

now if i update id:2 with passage_text_esp it insert the doc but does not generate embedding for it.

Upsert:

_bulk?pipeline=ingest-sample

{ “update” : { “_id” : “2”,“_index” : “sample_upsert”, “retry_on_conflict” : 2 } }
{ “doc” : { “passage_text_esp”:“Testing upsert insert”},“doc_as_upsert” : true}

Response:
{
“took”: 29,
“errors”: false,
“items”: [
{
“update”: {
“_index”: “sample_upsert”,
“_id”: “2”,
“_version”: 2,
“result”: “updated”,
“_shards”: {
“total”: 2,
“successful”: 1,
“failed”: 0
},
“_seq_no”: 6,
“_primary_term”: 1,
“status”: 200
}
}
]
}

Result:
{
“_index”: “sample_upsert”,
“_id”: “2”,
“_score”: 1,
“_source”: {
“passage_text”: “Vector search enables semantic similarity queries using embeddings.”,
“passage_embedding”: [0.026083095, -0.067727886, 0.043178726, -0.031542167, -0.004399777, -0.013967995, -0.060490288, 0.09509109, 0.00430261, -0.00627726, 0.080520645, -0.052798033, -0.049123146, -0.026538057, 0.014245002, -0.011685951, 0.0075974283, 0.13315636, -0.031493794, -0.05012921, 0.018730676, 0.030582182, 0.020599771, -0.01380502, -0.038501535, 0.016096147, -0.006807534, 0.013631381, 0.054272264, -0.0018038357, 0.12999037, 0.040789552, 0.06143593, 0.06944648, -0.057859898, -0.012377196, 0.001080881, 0.07147598, -0.04261844, -0.044861127, 0.027512576, 0.00792085, 0.030904537, 0.036124576, -0.025240015, 0.0715859, 0.031187076, -0.0062395963, 0.038677376, -0.019052638, 0.03661084, -0.043285843, -0.054633494, 0.04230883, 0.07051349, 0.049957406, 0.052673765, -0.06387623, -0.025792643, -0.12222278, -0.0063197757, 0.016614728, 0.017301125, 0.028382314, 0.025326028, -0.045545314, 0.042833623, 0.005541816, -0.024577096, 0.00968863, 0.028421259, 0.03309431, -0.078234784, 0.124440715, -0.11707664, 0.031267982, 0.05710486, -0.07583989, 0.035170246, -0.022910044, 0.011208475, 0.060569104, -0.009157271, -0.016050877, -0.03798157, -0.074460626, 0.10066296, -0.054388784, 0.06916367, -0.016868904, -0.033817463, -0.013092841, 0.026025236, -0.044133443, 0.026729885, -0.025777854, -0.013786166, 0.037714083, 0.061615895, 0.04515479, 0.030900449, 0.026327644, 0.0111779, 0.046724442, -0.022795724, 0.0006107711, -0.01332171, -0.058014993, 0.017698556, -0.055159174, 0.030442184, 0.08886656, -0.04125317, -0.08136491, 0.07030764, -0.05128093, -0.007745783, 0.12516777, 0.020515973, 0.045411535, -0.004825414, -0.0024048677, -0.051119573, 0.041593004, 0.038086798, -0.121287495, 0.012435926, 0.011650843, 0.02966544, 0.0059807114, 0.057140768, 0.06796318, -0.042273443, 0.070861585, -0.03928181, 0.119387485, -0.051519666, -0.11299718, -0.06892981, 0.06976501, -0.039368853, 0.0021159514, 0.058385145, 0.060598005, -0.098474056, 0.05302948, -0.16474278, -0.09827998, 0.031845506, -0.05129408, 0.014515105, -0.0406978, 0.02522653, 0.073249206, 0.042583324, 0.020570504, -0.006650351, -0.0072417734, -0.014669841, -0.047966473, -0.005201732, 0.053364657, -0.0025263058, -0.039364304, 0.0073737013, -0.017037245, -0.0941305, -0.07798143, -0.027609278, -0.028755449, 0.09940158, -0.028812764, -0.041964065, 0.060993083, -0.016161593, -0.023302216, -0.019840317, -0.057562355, 0.008787507, 0.008831341, 0.012702984, -0.02656768, 0.040744945, 0.03427152, -0.011669889, -0.009199524, 0.07497578, -0.028168956, -0.04926027, -0.035611432, 0.06821179, -0.065339506, 0.068973385, 0.05512504, 0.029516798, -0.019814305, 0.0833075, 0.01890894, 0.06179381, -0.016343502, -0.045608208, -0.11647018, -0.048096754, -0.03637659, -0.0331258, -0.026581155, 0.008210577, 0.02652634, -0.022560894, -0.08315273, -0.071950436, 0.013933145, -0.067799725, -0.029684471, 0.022390928, -0.113514, -0.026566021, -0.05366493, 0.027222464, 0.03472746, -0.017937606, -0.06020265, 0.029669115, -4.726142e-33, -0.079081506, -0.05615268, -0.011220467, 0.0074068503, -0.020694034, 0.031213587, 0.032043815, -0.0287803, -0.058529504, -0.05151516, -0.05462611, 0.027346218, -0.007450919, -0.009736384, -0.03331555, 0.09083842, -0.0024224175, 0.011424607, 0.022567078, 0.05948949, -0.014895368, -0.006669302, -0.07416872, 0.028840369, 0.0013264145, 0.026844021, 0.09908265, -0.04473576, 0.009974985, -0.027619805, -0.018952908, -0.07316988, -0.015699103, -0.008837521, -0.069700055, 0.02761229, 0.10022452, -0.008285672, -0.036324646, -0.070111185, 0.043701272, -0.016710376, -0.04223605, 0.0023628965, -0.11396539, 0.035824753, -0.009629503, 0.026583914, 0.07202662, 0.023823703, 0.0651506, 0.06373037, -0.06894256, -0.057089515, -0.07004105, 0.00563206, -0.07796538, 0.044103842, 0.00004574336, -0.042121634, -0.08946181, -0.034714896, -0.019962609, 0.053306773, -0.019965192, -0.02403357, 0.02275521, 0.059889223, -0.0137637695, -0.117569804, -0.0059109363, -0.10713167, 0.039872482, 0.01665636, -0.024587963, 0.061060756, 0.039661977, 0.039273452, -0.012802754, 0.07109534, -0.010605673, -0.012310071, -0.0039405106, -0.000557138, -0.006689474, 0.054849874, 0.033014473, 0.10975497, -0.020810908, -0.028282266, 0.021713153, -0.0015410905, 0.0006583933, 0.055106726, 0.026486602, 1.988666e-33, 0.0560179, 0.10378463, -0.03797907, -0.013366286, 0.03214722, -0.059459526, 0.02710635, 0.07157912, -0.04176391, 0.10804123, 0.02460433, 0.05546473, 0.011393352, -0.02736592, 0.038296975, -0.0067399833, 0.024409782, 0.0065370444, -0.012152865, -0.09282115, -0.027151264, 0.05831001, -0.014300941, 0.019107426, -0.01875739, 0.101735465, -0.06254901, 0.07654346, 0.1335275, 0.01948249, -0.0021897673, 0.026711788, -0.051988628, -0.03422898, 0.046197765, -0.013204699, -0.0085312035, -0.03628559, -0.059543137, -0.016776837, 0.0994159, -0.042206842, 0.042302284, -0.09695073, 0.073436245, 0.10136779, -0.013382648, -0.045513164, 0.031353094, -0.061408926, -0.004448006, -0.00274579, -0.000257434, 0.06641312, -0.02766623, -0.040209055, -0.07851905, 0.04211129, 0.002630921, 0.01265773, -0.0049386118, -0.00023492052, -0.0076999147, -0.07195595],
“id”: “2”,
“passage_text_esp”: “Testing upsert insert”
}
}
]
}

Please let me know if i have missed anything

1 Like
2

@tejashu Thank you for the question. There is already an open feature request for this here.

As a workaround you use semantic field type, more information is available here.

See below the devtools commands:

Update the previous model to add additional_config of space type = l2:

PUT /_plugins/_ml/models/<model_id>
{
  "model_config": {
    "model_type": "bert",
    "embedding_dimension": 384,
    "framework_type": "SENTENCE_TRANSFORMERS",
    "additional_config": {
      "space_type": "l2"
    }
  }
}

Create the index:

PUT /sample_upsert_v2
{
  "settings": {
    "index": {
      "knn": true
    }
  },
  "mappings": {
    "properties": {
      "id": { "type": "text" },
      "passage_text": {
        "type": "semantic",
        "model_id": "model_id>"
      },
      "passage_text_esp": {
        "type": "semantic",
        "model_id": "model_id>"
      }
    }
  }
}

Index and update the documents:

POST _bulk
{ "index": { "_index": "sample_upsert_v2", "_id": "1" } }
{ "passage_text": "OpenSearch is a distributed search and analytics engine built on Apache Lucene." }
{ "index": { "_index": "sample_upsert_v2", "_id": "2" } }
{ "id": "2", "passage_text": "Vector search enables semantic similarity queries using embeddings." }
{ "index": { "_index": "sample_upsert_v2", "_id": "3" } }
{ "id": "3", "passage_text": "HNSW is an efficient algorithm for approximate nearest neighbor search." }

POST _bulk
{ "update": { "_id": "2", "_index": "sample_upsert_v2", "retry_on_conflict": 2 } }
{ "doc": { "passage_text_esp": "Testing upsert insert" }, "doc_as_upsert": true }

Check the embeddings:

GET /sample_upsert_v2/_doc/2

You can also search without specifying the model as the model_id is already declared in the index mapping itself:

GET /sample_upsert_v2/_search
{
  "query": {
    "neural": {
      "passage_text": {
        "query_text": "approximate nearest neighbor algorithms",
        "k": 3
      }
    }
  }
}
3

Hi @Anthony ,
Thanks for the workaround .
In the workaround case, if we have to add chunking how can it be done for a field with semantic type?
Please let us know your thoughts.

Thanks,
Tejas

4

@tejashu You can enable chunking using the below example:

PUT /sample_semantic_chunked
{
  "settings": {
    "index": {
      "knn": true
    }
  },
  "mappings": {
    "properties": {
      "id": {
        "type": "text"
      },
      "passage_text": {
        "type": "semantic",
        "model_id": "l1UHcJwBy6v_8kBvclAa",
        "chunking": [
          {
            "algorithm": "fixed_token_length",
            "parameters": {
              "token_limit": 256,
              "overlap_rate": 0.2
            }
          }
        ]
      },
      "passage_text_esp": {
        "type": "semantic",
        "model_id": "l1UHcJwBy6v_8kBvclAa",
        "chunking": true
      }
    }
  }
}

The mapping should then contain the following:

"passage_text_esp_semantic_info": {
          "properties": {
            "chunks": {
              "type": "nested",
              "properties": {
                "embedding": {
                  "type": "knn_vector",
                  "dimension": 384,
                  "method": {
                    "engine": "faiss",
                    "space_type": "l2",
                    "name": "hnsw",
                    "parameters": {}
                  }
                },
                "text": {
                  "type": "text"
                }
              }
            },

You can then create and update the documents as usual:

POST _bulk
{ "index": { "_index": "sample_semantic_chunked", "_id": "1" } }
{ "passage_text": "OpenSearch is a distributed search and analytics engine built on Apache Lucene. It provides powerful full-text search capabilities and is designed for scalability and performance." }
{ "index": { "_index": "sample_semantic_chunked", "_id": "2" } }
{ "id": "2", "passage_text": "Vector search enables semantic similarity queries using embeddings. This allows you to find documents based on meaning rather than exact keyword matches. Machine learning models convert text into dense vectors." }
{ "index": { "_index": "sample_semantic_chunked", "_id": "3" } }
{ "id": "3", "passage_text": "HNSW is an efficient algorithm for approximate nearest neighbor search. It creates a hierarchical graph structure for fast vector similarity search across millions of documents." }

POST _bulk
{ "update": { "_id": "2", "_index": "sample_semantic_chunked", "retry_on_conflict": 2 } }
{ "doc": { "passage_text_esp": "Testing upsert with automatic chunking. This text will be split into chunks and each chunk will get its own embedding vector. The semantic field handles all of this automatically during the update operation without needing any ingest pipeline." }, "doc_as_upsert": true }
{ "update": { "_id": "4", "_index": "sample_semantic_chunked", "retry_on_conflict": 2 } }
{ "doc": { "id": "4", "passage_text": "This is a new document created via upsert. Lucene provides the underlying engine for indexing and search.", "passage_text_esp": "New content with chunking enabled for testing purposes." }, "doc_as_upsert": true }

and verify with:

GET /sample_semantic_chunked/_doc/2

"_source": {
    "passage_text": "Vector search enables semantic similarity queries using embeddings. This allows you to find documents based on meaning rather than exact keyword matches. Machine learning models convert text into dense vectors.",
    "passage_text_semantic_info": {
      "chunks": [
        {
          "text": "Vector search enables semantic similarity queries using embeddings. This allows you to find documents based on meaning rather than exact keyword matches. Machine learning models convert text into dense vectors.",
          "embedding": [
            -0.02185014,
            0.006674238,
            ...
5

Thanks,
Is this only available on Faiss engine or can it be changed to lucene?

6

@tejashu if the semantic field is used then the method defaults to Faiss, setting this manually to Lucene would produce an error. This seems to be one of the limitations currently if you need to use update.

7

thanks i am seeing storage difference? Is it also because of Faiss engine?

8

Hi @Anthony could you share what is the error you have seen with Lucene engine. Semantic field should support Lucene engine.

9

@heemin When I run the below command:

POST /_plugins/_ml/models/_register
{
“name”: “huggingface/sentence-transformers/all-MiniLM-L6-v2”,
“version”: “1.0.1”,
“model_format”: “TORCH_SCRIPT”,
“model_config”: {
“model_type”: “bert”,
“embedding_dimension”: 384,
“framework_type”: “SENTENCE_TRANSFORMERS”,
}
}

PUT /_plugins/_ml/models/0fZ215wBney3m3UZWoMh
{
“model_config”: {
“model_type”: “bert”,
“embedding_dimension”: 384,
“framework_type”: “SENTENCE_TRANSFORMERS”,
“additional_config”: {
“space_type”: “l2”
}
}
}

PUT /sample_semantic_lucene
{
“settings”: { “index”: { “knn”: true } },
“mappings”: {
“properties”: {
“passage_text”: {
“type”: “semantic”,
“model_id”: “0fZ215wBney3m3UZWoMh”,
“method”: {
“engine”: “lucene”,
“name”: “hnsw”,
“space_type”: “l2”
}
}
}
}
}

I get an error with the following logs:

opensearch-node3_3.3.0       | org.opensearch.index.mapper.MapperParsingException: Failed to parse mapping [_doc]: unknown parameter [method] on mapper [passage_text] of type [text]
opensearch-node3_3.3.0       | 	at org.opensearch.index.mapper.MapperService.internalMerge(MapperService.java:484) ~[opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.index.mapper.MapperService.merge(MapperService.java:452) ~[opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.cluster.metadata.MetadataCreateIndexService.updateIndexMappingsAndBuildSortOrder(MetadataCreateIndexService.java:1435) ~[opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.cluster.metadata.MetadataCreateIndexService.lambda$applyCreateIndexWithTemporaryService$0(MetadataCreateIndexService.java:534) [opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.indices.IndicesService.withTempIndexService(IndicesService.java:1026) [opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.cluster.metadata.MetadataCreateIndexService.applyCreateIndexWithTemporaryService(MetadataCreateIndexService.java:525) [opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.cluster.metadata.MetadataCreateIndexService.applyCreateIndexRequestWithV1Templates(MetadataCreateIndexService.java:735) [opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.cluster.metadata.MetadataCreateIndexService.applyCreateIndexRequest(MetadataCreateIndexService.java:487) [opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.cluster.metadata.MetadataCreateIndexService.applyCreateIndexRequest(MetadataCreateIndexService.java:494) [opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.cluster.metadata.MetadataCreateIndexService$1.execute(MetadataCreateIndexService.java:394) [opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.cluster.ClusterStateUpdateTask.execute(ClusterStateUpdateTask.java:67) [opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.cluster.service.ClusterManagerService.executeTasks(ClusterManagerService.java:890) [opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.cluster.service.ClusterManagerService.calculateTaskOutputs(ClusterManagerService.java:441) [opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.cluster.service.ClusterManagerService.runTasks(ClusterManagerService.java:301) [opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.cluster.service.ClusterManagerService$Batcher.run(ClusterManagerService.java:214) [opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:206) [opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:264) [opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:916) [opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.common.util.concurrent.PrioritizedOpenSearchThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedOpenSearchThreadPoolExecutor.java:299) [opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.common.util.concurrent.PrioritizedOpenSearchThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedOpenSearchThreadPoolExecutor.java:262) [opensearch-3.3.0.jar:3.3.0]
opensearch-dashboards_3.3.0  | {"type":"response","@timestamp":"2026-03-10T11:24:49Z","tags":["access:console"],"pid":1,"method":"post","statusCode":400,"req":{"url":"/api/console/proxy?path=%2Fsample_semantic_lucene&method=PUT&dataSourceId=","routePath":"/api/console/proxy","method":"post","headers":{"host":"localhost:5601","connection":"keep-alive","content-length":"313","osd-version":"3.3.0","sec-ch-ua-platform":"\"macOS\"","user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36","sec-ch-ua":"\"Not:A-Brand\";v=\"99\", \"Google Chrome\";v=\"145\", \"Chromium\";v=\"145\"","content-type":"application/json","sec-ch-ua-mobile":"?0","osd-xsrf":"osd-fetch","accept":"*/*","origin":"https://localhost:5601","sec-fetch-site":"same-origin","sec-fetch-mode":"cors","sec-fetch-dest":"empty","referer":"https://localhost:5601/app/dev_tools","accept-encoding":"gzip, deflate, br, zstd","accept-language":"en-US,en;q=0.9","securitytenant":"__user__"},"remoteAddress":"192.168.65.1","userAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36","referer":"https://localhost:5601/app/dev_tools"},"res":{"statusCode":400,"responseTime":100,"contentLength":9},"message":"POST /api/console/proxy?path=%2Fsample_semantic_lucene&method=PUT&dataSourceId= 400 100ms - 9.0B"}
opensearch-node3_3.3.0       | 	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095) [?:?]

opensearch-node3_3.3.0       | 	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619) [?:?]
opensearch-node3_3.3.0       | 	at java.base/java.lang.Thread.run(Thread.java:1447) [?:?]
opensearch-node3_3.3.0       | Caused by: org.opensearch.index.mapper.MapperParsingException: unknown parameter [method] on mapper [passage_text] of type [text]
opensearch-node3_3.3.0       | 	at org.opensearch.index.mapper.ParametrizedFieldMapper$Builder.parse(ParametrizedFieldMapper.java:715) ~[opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.index.mapper.ParametrizedFieldMapper$TypeParser.parse(ParametrizedFieldMapper.java:769) ~[opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.neuralsearch.mapper.SemanticFieldMapper$TypeParser.parse(SemanticFieldMapper.java:297) ~[?:?]
opensearch-node3_3.3.0       | 	at org.opensearch.neuralsearch.mapper.SemanticFieldMapper$TypeParser.parse(SemanticFieldMapper.java:268) ~[?:?]
opensearch-node3_3.3.0       | 	at org.opensearch.index.mapper.ObjectMapper$TypeParser.parseProperties(ObjectMapper.java:585) ~[opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.index.mapper.ObjectMapper$TypeParser.parseObjectOrDocumentTypeProperties(ObjectMapper.java:342) ~[opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.index.mapper.RootObjectMapper$TypeParser.parse(RootObjectMapper.java:193) ~[opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.index.mapper.DocumentMapperParser.parse(DocumentMapperParser.java:143) ~[opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.index.mapper.DocumentMapperParser.parse(DocumentMapperParser.java:132) ~[opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	at org.opensearch.index.mapper.MapperService.internalMerge(MapperService.java:482) ~[opensearch-3.3.0.jar:3.3.0]
opensearch-node3_3.3.0       | 	... 22 more
10

Hi @Anthony, I think you are using the incorrect parameter to configure the semantic field. You can use the dense_embedding_config to configure to use the lucene engine - Semantic - OpenSearch Documentation

11

@bozhang You are correct, thank you for pointing this out, the following can be used:

PUT /sample_semantic_lucene
{
  "settings": { "index": { "knn": true } },
  "mappings": {
    "properties": {
      "passage_text": {
        "type": "semantic",
        "model_id": "szwa2ZwBMlEnbBThfQwC",
        "dense_embedding_config": {
          "method": {
            "engine": "lucene",
            "name": "hnsw"
          }
        }
      }
    }
  }
}