From 88d7cb47e2c1b5a1082b9d9db3aafbf463b939ec Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Wed, 18 Oct 2023 13:39:23 +0100 Subject: [PATCH] Revert "Bugfix/OpenSearch illegal invocation" --- .../nodes/vectorstores/OpenSearch/core.ts | 8 --- .../OpenSearch_existing.ts | 52 +----------------- .../opensearch.png | Bin .../OpenSearch_Upsert.ts | 40 +------------- .../OpenSearch_Upsert/opensearch.png | Bin 0 -> 5216 bytes 5 files changed, 3 insertions(+), 97 deletions(-) delete mode 100644 packages/components/nodes/vectorstores/OpenSearch/core.ts rename packages/components/nodes/vectorstores/{OpenSearch => OpenSearch_Existing}/OpenSearch_existing.ts (62%) rename packages/components/nodes/vectorstores/{OpenSearch => OpenSearch_Existing}/opensearch.png (100%) rename packages/components/nodes/vectorstores/{OpenSearch => OpenSearch_Upsert}/OpenSearch_Upsert.ts (73%) create mode 100644 packages/components/nodes/vectorstores/OpenSearch_Upsert/opensearch.png diff --git a/packages/components/nodes/vectorstores/OpenSearch/core.ts b/packages/components/nodes/vectorstores/OpenSearch/core.ts deleted file mode 100644 index 49a1d6e3..00000000 --- a/packages/components/nodes/vectorstores/OpenSearch/core.ts +++ /dev/null @@ -1,8 +0,0 @@ -export const buildMetadataTerms = (filter?: object): { term: Record }[] => { - if (filter == null) return [] - const result = [] - for (const [key, value] of Object.entries(filter)) { - result.push({ term: { [`metadata.${key}`]: value } }) - } - return result -} diff --git a/packages/components/nodes/vectorstores/OpenSearch/OpenSearch_existing.ts b/packages/components/nodes/vectorstores/OpenSearch_Existing/OpenSearch_existing.ts similarity index 62% rename from packages/components/nodes/vectorstores/OpenSearch/OpenSearch_existing.ts rename to packages/components/nodes/vectorstores/OpenSearch_Existing/OpenSearch_existing.ts index aeab0e3f..c8d09470 100644 --- a/packages/components/nodes/vectorstores/OpenSearch/OpenSearch_existing.ts +++ b/packages/components/nodes/vectorstores/OpenSearch_Existing/OpenSearch_existing.ts @@ -1,10 +1,8 @@ import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' import { OpenSearchVectorStore } from 'langchain/vectorstores/opensearch' import { Embeddings } from 'langchain/embeddings/base' -import { Document } from 'langchain/document' -import { Client, RequestParams } from '@opensearch-project/opensearch' +import { Client } from '@opensearch-project/opensearch' import { getBaseClasses } from '../../../src/utils' -import { buildMetadataTerms } from './core' class OpenSearch_Existing_VectorStores implements INode { label: string @@ -44,13 +42,6 @@ class OpenSearch_Existing_VectorStores implements INode { name: 'indexName', type: 'string' }, - { - label: 'OpenSearch Metadata Filter', - name: 'openSearchMetadataFilter', - type: 'json', - optional: true, - additionalParams: true - }, { label: 'Top K', name: 'topK', @@ -82,7 +73,6 @@ class OpenSearch_Existing_VectorStores implements INode { const output = nodeData.outputs?.output as string const topK = nodeData.inputs?.topK as string const k = topK ? parseFloat(topK) : 4 - const openSearchMetadataFilter = nodeData.inputs?.openSearchMetadataFilter const client = new Client({ nodes: [opensearchURL] @@ -93,46 +83,6 @@ class OpenSearch_Existing_VectorStores implements INode { indexName }) - vectorStore.similaritySearchVectorWithScore = async ( - query: number[], - k: number, - filter?: object | undefined - ): Promise<[Document, number][]> => { - if (openSearchMetadataFilter) { - const metadatafilter = - typeof openSearchMetadataFilter === 'object' ? openSearchMetadataFilter : JSON.parse(openSearchMetadataFilter) - filter = metadatafilter - } - const search: RequestParams.Search = { - index: indexName, - body: { - query: { - bool: { - filter: { bool: { must: buildMetadataTerms(filter) } }, - must: [ - { - knn: { - embedding: { vector: query, k } - } - } - ] - } - }, - size: k - } - } - - const { body } = await client.search(search) - - return body.hits.hits.map((hit: any) => [ - new Document({ - pageContent: hit._source.text, - metadata: hit._source.metadata - }), - hit._score - ]) - } - if (output === 'retriever') { const retriever = vectorStore.asRetriever(k) return retriever diff --git a/packages/components/nodes/vectorstores/OpenSearch/opensearch.png b/packages/components/nodes/vectorstores/OpenSearch_Existing/opensearch.png similarity index 100% rename from packages/components/nodes/vectorstores/OpenSearch/opensearch.png rename to packages/components/nodes/vectorstores/OpenSearch_Existing/opensearch.png diff --git a/packages/components/nodes/vectorstores/OpenSearch/OpenSearch_Upsert.ts b/packages/components/nodes/vectorstores/OpenSearch_Upsert/OpenSearch_Upsert.ts similarity index 73% rename from packages/components/nodes/vectorstores/OpenSearch/OpenSearch_Upsert.ts rename to packages/components/nodes/vectorstores/OpenSearch_Upsert/OpenSearch_Upsert.ts index 6cb77414..c11d8b11 100644 --- a/packages/components/nodes/vectorstores/OpenSearch/OpenSearch_Upsert.ts +++ b/packages/components/nodes/vectorstores/OpenSearch_Upsert/OpenSearch_Upsert.ts @@ -2,10 +2,9 @@ import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/I import { OpenSearchVectorStore } from 'langchain/vectorstores/opensearch' import { Embeddings } from 'langchain/embeddings/base' import { Document } from 'langchain/document' -import { Client, RequestParams } from '@opensearch-project/opensearch' +import { Client } from '@opensearch-project/opensearch' import { flatten } from 'lodash' import { getBaseClasses } from '../../../src/utils' -import { buildMetadataTerms } from './core' class OpenSearchUpsert_VectorStores implements INode { label: string @@ -96,44 +95,9 @@ class OpenSearchUpsert_VectorStores implements INode { const vectorStore = await OpenSearchVectorStore.fromDocuments(finalDocs, embeddings, { client, - indexName + indexName: indexName }) - vectorStore.similaritySearchVectorWithScore = async ( - query: number[], - k: number, - filter?: object | undefined - ): Promise<[Document, number][]> => { - const search: RequestParams.Search = { - index: indexName, - body: { - query: { - bool: { - filter: { bool: { must: buildMetadataTerms(filter) } }, - must: [ - { - knn: { - embedding: { vector: query, k } - } - } - ] - } - }, - size: k - } - } - - const { body } = await client.search(search) - - return body.hits.hits.map((hit: any) => [ - new Document({ - pageContent: hit._source.text, - metadata: hit._source.metadata - }), - hit._score - ]) - } - if (output === 'retriever') { const retriever = vectorStore.asRetriever(k) return retriever diff --git a/packages/components/nodes/vectorstores/OpenSearch_Upsert/opensearch.png b/packages/components/nodes/vectorstores/OpenSearch_Upsert/opensearch.png new file mode 100644 index 0000000000000000000000000000000000000000..3fdcfd3f09ed02c1478f690884ecafe844cb32ae GIT binary patch literal 5216 zcmcgwS5y-~vkpy=5~`pOy7b-ys0f56y@o0sX-Wtkq)6`&dIt?vq=&9_fl!pDAT0=? zO7955pa1bb+{b&*p4mD3&6(Yu**UYb8>gqEMnTF*3IG5oG}IvmIF9-+k>KO{#yW%} zjuF06)K&xl8dAt^Y>9AVHhXmgZ2%yU8vqE41OWcxps+0fz(*7S*s%csVPtE_FT+hti>ComH1eO{D|CTb80SS} z1kh=M2X3F`ll>ayrBi_#S~O`Bo^o0=_)CTNNEvwEs_y)oyP@YNoP{UN+3{qGGt1 z1q^{vN}i{`zvzOl<5c%}?3z@YuHI@~Z6|OcY12_LyhH+e$uZZ{f3?fb zM{s|QPvZoO9#YnIr#)ogRI~;SmDOqU&vlOK*0^r(`hr0yHIZimYQRi`t4`=kEnST) zlwl^^&}1-qQD|>AkjLYpb0{%K6C4W;__XrAaG}bZnGxh|a{;U|S(;gHP6M(FK$0^C z+Mw3{s_zeXvVJhpk-iC5HQ(PjgYM+N5)#1jJ49&}E1X>Fm!;jBiW{V1ZYyS`9bZiV zAs~}J1ZVK;sV0T=QHpjG8S*-)}izSw5Z%tPfksPGK!$j)Go(2K|wk#`lq%*B2Z|5?vQxUwMiY z@9GU~uFs|Bs>%~WK|%hQ>6AN)J>DlcP+eQ6gtoV=o8z9es`E2qEQ9(%2B zWTy^pu7^!)$m9EI#HR%R6|?SWKMRoEB_l&AQ}rPJU11wjAw!$%qPeeJV%Px5Yau6# zwCsxSPz0W2#o6}m@OWR6J`zfbfSpNB6cUERke#8p6~bk6x;eSTihSut*G}Lw&8TUh zq9|$9>Q#EmRB~Su*oAmS2)dIu*3{+wt6})(nc0Gu^mMY_qqi@U1FCE4+Y){0iVm?z zHHTX*9VOXS0}`p5(Bfi=huTvVkjDlzYjEk2KpiW7S#QPR5w4k*0N?X*h3-^}mLttZVCnF4jc27!&-l?qfO{iAZK5k+jnL(L*-hJx^Qhw;0=X z^5#1Qy-sAA8XZG8x|x&LGWZ?Wf83jYCz$kDE%}-}fzW4yHON1N(y*P^CKt=kFj&ulvcnZ3A09rxHm2Jw1el#|mV zLK`(dNrV5b3pC5s_`X&uxcM$lMn<;&HxPIFc($_m;SCprbfkbBMiq~02LhbAj#55l z#(_2Fw5&`*3(7KKQu@>t-x$y^W-eP<_%R|U4U)k{{kZXIeND=k~5 zUybZrcSIj<`P*CY?D6hLmx7L}<{D3~S)1i%vU(IHWQLx&x$Nz<>zR9YR65xBJJ|M8 zKVsR~RJ%*GZ(0z^G@edgeWM;ClItf7SpyH$r5`u;6Wv3$*Oz{-FSRT)T24pgwK^&< z51#R9%&;I|m6gFjJgxGAc4|PKcfl#Wy1_#tHD2O~HF{u$m$zMGOVydT{&wz%siJr9 ziDEd#1&#K2T)6A!zt@$%5NRE$7{HpQ`wz)FB2r;9R?&vO+3N{orZ?Je?K{2=*FMv`DAa=l#Bt6%8c2Ip(0r{#5a=QTaGCiL=4=HhP_PlRUVTw&bgn(LL>`6(^ zl88P|*V;^jPl^1NA-u(aK`!Kl203-DtPP;zG?(a1x;B>r=$y`r)MszZ{;zKZ=$; zyv>sYe}?Uy?Ym8dt~H&IEtKMLJgLI%ro8XuR29yo&#BO=zGP+aQLDg)aCt}Vw(ddb z=iP;S(W-=!JF97-I;c`Aakzi6`WNo|L)LxoE2!vI9xJ(AeLl>e;{WpJ;*^3+V*@5; zCdy;3NGc9Dg`~;&aO!l$_kU+oof>afKRma--RVeHFD}tyg{lGBXN85?Ib*h?owYqd z2K&8n*`B-5;W>P+e-1X^bEF?)u2v+vph8|=$nf}lqL`yDNr8Ens4>6SdGy`mmqr5> zeRiq~#@;_|QJ5J1RAG2m+wSn~Q%&J&57z^@TqHRUIQ)X^sOzn`8t{aIV;ujGa^R;G z_(;4ckhzmp(UKI4x*3qtsW?H_eG|D#TN!Z@E>}$YkO|HB=Mt4tLgl_3wf=`fu#LEt>!w&dxH%~qo{K^5azYVk-0x!fA}aCm4iFZ*#r31{6%4d$ zK$RU%Oe(sVkR`D75t)gHQFsE^Og{PQ*q1fTPZzt6;qfWe9RSCn#b&gBH}(s4H9fwg z_LhDgdfyO9qb4fmyT$xO5VCBs4HW%oAL?0Xn#tYPGF(vjrD=0u;PtAY@Ka3Yfxc7pKO>-N;Pw(dHjwW=MMiS@= z@|Q?^w!K3FO;*ea0qC0_>$MlLOgtL0f9Gv{1inW-1I6rG7Tm}`)GI0EMfv>Zk&a_mi&7z@h8nV(J_8jP_?2JsskE zZ`FYSAN;zO%6?D(mHJpsg)B(L5BjXE3(EV&>BB454sugGcHBc8k?T1R>8)PzT~4MT zIuMp8x8~zPEJm)gMC?VO(Z~vwQ!u*1^`#K|a?~Yk&Z}>!%&d=3kdO9lIzF5HcVkzf zcM{+L*}n^7TpOrD$mO5mLQ7of@Z`$Wm*qec`R}m_b9#3FHsq`9>X%fTf5EjsaSsxu zkJTzng^-W-nvM7?=Ct>o{rpBll27%vwjqxWr)FE*? z3=A5}Y1DM_?zekKTGLyvIcJ(8;ihA4m+UKsd}{;yPDHSS5Ga|rt0uazuFNWJ9+NCr zJWpVv7=(VKURvEh_&I6|ZGPOFU7YnSv@a&0q{p~9YLIk}WJ5qhIA$rj>A-H8!>7B5 z-8C@MhGxoge05<%U(@$IchJ_$mHZV3*?AP-8QZ#VBuYx;%j$U+-C;1rH=XgVilFdv zgI3Op%q}p|e0G5aLU@;i@&Psvj51GmuDI4l-_kTMFGc>mNBJHx95noc>7z*eZP;y$ z-WNs@`3Tp52i{%zii|;)Fw)(iMPsXGrnSm!4*lr!8uN8oqx*ad0$eBbi4(j%g{^+M zA62uvPwRK`o;#1?m3*W!jzUG4#($F|g^r79wE1;c%bd2P4gWEtc{GeyvV$N0EN!Ds z5pb6Feb&VXd`SkC@Q@aWSD>9J@f%hK0>+ImcVCZ@R#mEHxFs1_C&59$BFpky*p(Tz zfS_Q4I~D-@!ob=U*MK+g2_Pe*qsj0EnR5p_v@uZ>85v(Z>|4kr`zjZc`RU1oT$27IaB*YpUP86hCM` z71Nq3)T2lI!WBgls*+HGo;(Q2l3O2Bx$4n+Xs=Ud?NA#X-4lXYi%Uu9`*QV;)Gp;( zH~=OmIv4em@zRCr5s!*(8``$uZ+a=^0>|3n@X6>5$3xb>Ul)}#JZTM}1k=sQ>G0Zn zTj97XLl3cR^7ic2{f*LY9yz>}CzLhcegp|hfX`S_EWpv1$tr>pTc6Y{DCl=d66gcI zhAqW6*6IBvB79T4_P~>GG{2!fOyVxd(={l^Ygk^j+JjR6V`>A@wnb{r{%b}13fk9F zl}|)`&U%~8{4de#bbhO=G0qQ6g|&>{>LpuTRv|Lte3(+}fr|JSwc?K^s??ODGQ3{M z{HkZZQp@hvuj4Phok3@>w=pLve4wR%kP<63LDZ`NxP5m>8P<x9OGj@6jLB6yZQ*3Xp>eWk3-#|xM0eb`Ey z9vM^oJe8^1reo*m%zEACUJhvBH`sl-U5n;QXub}&*JE_8BRuu7B+WI8pPxeGAe=6g zJKcVeLklR*OE-)j3MO9c{N?*$2mhnU$OyTpY;0<3^pcw8dNSr)AHHxf#P3-IC3j3^ zBxLm+KHk~Dh7;^uq*3T)AC@YvL=TI#@At(^Z&_0Yp9n{pfm?0i{@TuTxFWxh$Gy68 z!<8cPU|l;XAsAHQOkC3B;B z;%WEaAUKJ0Z{@;9pgnm&F)Jjv3yq(pfiG6$Sh`_r5Zv`NX`g$Z;3U$RJEoYCl@aG6O&xfFDjp7r#P67?Y6eB<<0g$uW~X@`kH;;S*5yMzZY=(Ac7ZT1xg|@yXV= z*p(nk^fx7fS4x5Z3P5=LF;mB5789oL=4o=>4R^SzpC>N{8|2vLsaoC7`xs{eOV2T8 zQyZiYPm%|1xu|Gz1und83~ALgc?ZH{hvYFcJy&}=f7S)f`t=$ImAgc$#!qKVze9wa zN+O%^?j2poAeeA-o>#YdbVG%-Qf%9Z|1ocQT%0jcDSpD2b&bbG>6_YRj&eRN%ONap z?S)-wHoVF_@KbWx{TC157i<#5z0`m=%X+4pq6);LTO`_#|DFY zIJvkxvUvx<9NAo8-u3{1f5F-@If