From f108c62acf009d4b0ef26690cdffe0f18804b6ea Mon Sep 17 00:00:00 2001
From: vinodkiran <vinodkiran@usa.net>
Date: Thu, 28 Sep 2023 10:31:40 +0530
Subject: [PATCH 1/3] Support for ElasticSearch as a vector store

---
 .../ElasticsearchAPI.credential.ts            |  31 ++++
 .../ElectricsearchUserPassword.credential.ts  |  31 ++++
 .../Elasticsearch/Elasticsearch_Existing.ts   | 111 ++++++++++++
 .../Elasticsearch/Elasticsearch_Upsert.ts     | 165 ++++++++++++++++++
 .../Elasticsearch/elasticsearch.png           | Bin 0 -> 3719 bytes
 packages/components/package.json              |   1 +
 6 files changed, 339 insertions(+)
 create mode 100644 packages/components/credentials/ElasticsearchAPI.credential.ts
 create mode 100644 packages/components/credentials/ElectricsearchUserPassword.credential.ts
 create mode 100644 packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Existing.ts
 create mode 100644 packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts
 create mode 100644 packages/components/nodes/vectorstores/Elasticsearch/elasticsearch.png

diff --git a/packages/components/credentials/ElasticsearchAPI.credential.ts b/packages/components/credentials/ElasticsearchAPI.credential.ts
new file mode 100644
index 00000000..e377243d
--- /dev/null
+++ b/packages/components/credentials/ElasticsearchAPI.credential.ts
@@ -0,0 +1,31 @@
+import { INodeParams, INodeCredential } from '../src/Interface'
+
+class ElectricsearchAPI implements INodeCredential {
+    label: string
+    name: string
+    version: number
+    description: string
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'Elasticsearch API'
+        this.name = 'elasticsearchApi'
+        this.version = 1.0
+        this.description =
+            'Refer to <a target="_blank" href="https://www.elastic.co/guide/en/kibana/current/api-keys.html">official guide</a> on how to get an API Key from ElasticSearch'
+        this.inputs = [
+            {
+                label: 'Elasticsearch Endpoint',
+                name: 'endpoint',
+                type: 'string'
+            },
+            {
+                label: 'Elasticsearch API ID',
+                name: 'apiKey',
+                type: 'password'
+            }
+        ]
+    }
+}
+
+module.exports = { credClass: ElectricsearchAPI }
diff --git a/packages/components/credentials/ElectricsearchUserPassword.credential.ts b/packages/components/credentials/ElectricsearchUserPassword.credential.ts
new file mode 100644
index 00000000..2dd88937
--- /dev/null
+++ b/packages/components/credentials/ElectricsearchUserPassword.credential.ts
@@ -0,0 +1,31 @@
+import { INodeParams, INodeCredential } from '../src/Interface'
+
+class ElasticSearchUserPassword implements INodeCredential {
+    label: string
+    name: string
+    version: number
+    description: string
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'ElasticSearch User Password'
+        this.name = 'elasticSearchUserPassword'
+        this.version = 1.0
+        this.description =
+            'Refer to <a target="_blank" href="https://www.elastic.co/guide/en/kibana/current/tutorial-secure-access-to-kibana.html">official guide</a> on how to get User Password from ElasticSearch'
+        this.inputs = [
+            {
+                label: 'ElasticSearch User',
+                name: 'elasticSearchUser',
+                type: 'string'
+            },
+            {
+                label: 'ElasticSearch Password',
+                name: 'elasticSearchPassword',
+                type: 'password'
+            }
+        ]
+    }
+}
+
+module.exports = { credClass: ElasticSearchUserPassword }
diff --git a/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Existing.ts b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Existing.ts
new file mode 100644
index 00000000..6e785c85
--- /dev/null
+++ b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Existing.ts
@@ -0,0 +1,111 @@
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { Embeddings } from 'langchain/embeddings/base'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src'
+
+import { Client, ClientOptions } from '@elastic/elasticsearch'
+import { ElasticClientArgs, ElasticVectorSearch } from 'langchain/vectorstores/elasticsearch'
+
+class ElasicsearchExisting_VectorStores implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    credential: INodeParams
+    outputs: INodeOutputsValue[]
+
+    constructor() {
+        this.label = 'Elasticsearch Load Existing Index'
+        this.name = 'ElasticsearchIndex'
+        this.version = 1.0
+        this.type = 'Elasticsearch'
+        this.icon = 'elasticsearch.png'
+        this.category = 'Vector Stores'
+        this.description = 'Load existing index from Elasticsearch   (i.e: Document has been upserted)'
+        this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['elasticsearchApi', 'elasticSearchUserPassword']
+        }
+        this.inputs = [
+            {
+                label: 'Embeddings',
+                name: 'embeddings',
+                type: 'Embeddings'
+            },
+            {
+                label: 'Index Name',
+                name: 'indexName',
+                placeholder: '<INDEX_NAME>',
+                type: 'string'
+            },
+            {
+                label: 'Top K',
+                name: 'topK',
+                description: 'Number of top results to fetch. Default to 4',
+                placeholder: '4',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            }
+        ]
+        this.outputs = [
+            {
+                label: 'Elasticsearch Retriever',
+                name: 'retriever',
+                baseClasses: this.baseClasses
+            },
+            {
+                label: 'Elasticsearch Vector Store',
+                name: 'vectorStore',
+                baseClasses: [this.type, ...getBaseClasses(ElasticVectorSearch)]
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const endPoint = getCredentialParam('endpoint', credentialData, nodeData)
+        const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
+        const indexName = nodeData.inputs?.indexName as string
+        const embeddings = nodeData.inputs?.embeddings as Embeddings
+        const topK = nodeData.inputs?.topK as string
+
+        const k = topK ? parseFloat(topK) : 4
+        const output = nodeData.outputs?.output as string
+
+        // eslint-disable-next-line no-console
+        console.log('EndPoint:: ' + endPoint + ', APIKey:: ' + apiKey + ', Index:: ' + indexName)
+
+        const elasticSearchClientOptions: ClientOptions = {
+            node: endPoint,
+            auth: {
+                apiKey: apiKey
+            }
+        }
+
+        const elasticSearchClientArgs: ElasticClientArgs = {
+            client: new Client(elasticSearchClientOptions),
+            indexName: indexName
+        }
+
+        const vectorStore = await ElasticVectorSearch.fromExistingIndex(embeddings, elasticSearchClientArgs)
+        // eslint-disable-next-line no-console
+        console.log('vectorStore ::' + vectorStore._vectorstoreType())
+        if (output === 'retriever') {
+            return vectorStore.asRetriever(k)
+        } else if (output === 'vectorStore') {
+            ;(vectorStore as any).k = k
+            return vectorStore
+        }
+        return vectorStore
+    }
+}
+
+module.exports = { nodeClass: ElasicsearchExisting_VectorStores }
diff --git a/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts
new file mode 100644
index 00000000..5a0065d5
--- /dev/null
+++ b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts
@@ -0,0 +1,165 @@
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { Embeddings } from 'langchain/embeddings/base'
+import { Document } from 'langchain/document'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src'
+
+import { Client, ClientOptions } from '@elastic/elasticsearch'
+import { ElasticClientArgs, ElasticVectorSearch } from 'langchain/vectorstores/elasticsearch'
+import { flatten } from 'lodash'
+
+class ElasicsearchUpsert_VectorStores implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    credential: INodeParams
+    outputs: INodeOutputsValue[]
+
+    constructor() {
+        this.label = 'Elasticsearch Upsert Document'
+        this.name = 'ElasticsearchUpsert'
+        this.version = 1.0
+        this.type = 'Elasticsearch'
+        this.icon = 'elasticsearch.png'
+        this.category = 'Vector Stores'
+        this.description = 'Upsert documents to Elasticsearch'
+        this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['elasticsearchApi', 'elasticSearchUserPassword']
+        }
+        this.inputs = [
+            {
+                label: 'Document',
+                name: 'document',
+                type: 'Document',
+                list: true
+            },
+            {
+                label: 'Embeddings',
+                name: 'embeddings',
+                type: 'Embeddings'
+            },
+            {
+                label: 'Index Name',
+                name: 'indexName',
+                placeholder: '<INDEX_NAME>',
+                type: 'string'
+            },
+            {
+                label: 'Top K',
+                name: 'topK',
+                description: 'Number of top results to fetch. Default to 4',
+                placeholder: '4',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Similarity',
+                name: 'similarity',
+                description: 'Similarity measure used in Elasticsearch.',
+                type: 'options',
+                default: 'l2_norm',
+                options: [
+                    {
+                        label: 'l2_norm',
+                        name: 'l2_norm'
+                    },
+                    {
+                        label: 'dot_product',
+                        name: 'dot_product'
+                    },
+                    {
+                        label: 'cosine',
+                        name: 'cosine'
+                    }
+                ],
+                additionalParams: true,
+                optional: true
+            }
+        ]
+        this.outputs = [
+            {
+                label: 'Elasticsearch Retriever',
+                name: 'retriever',
+                baseClasses: this.baseClasses
+            },
+            {
+                label: 'Elasticsearch Vector Store',
+                name: 'vectorStore',
+                baseClasses: [this.type, ...getBaseClasses(ElasticVectorSearch)]
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const endPoint = getCredentialParam('endpoint', credentialData, nodeData)
+        const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
+        const docs = nodeData.inputs?.document as Document[]
+        const indexName = nodeData.inputs?.indexName as string
+        const embeddings = nodeData.inputs?.embeddings as Embeddings
+        const topK = nodeData.inputs?.topK as string
+        const k = topK ? parseFloat(topK) : 4
+        const output = nodeData.outputs?.output as string
+        const similarityMeasure = nodeData.inputs?.similarityMeasure as string
+
+        // eslint-disable-next-line no-console
+        console.log('EndPoint:: ' + endPoint + ', APIKey:: ' + apiKey + ', Index:: ' + indexName)
+
+        const elasticSearchClientOptions: ClientOptions = {
+            node: endPoint,
+            auth: {
+                apiKey: apiKey
+            }
+        }
+        let vectorSearchOptions = {}
+        switch (similarityMeasure) {
+            case 'dot_product':
+                vectorSearchOptions = {
+                    similarity: 'dot_product'
+                }
+                break
+            case 'cosine':
+                vectorSearchOptions = {
+                    similarity: 'cosine'
+                }
+                break
+            default:
+                vectorSearchOptions = {
+                    similarity: 'l2_norm'
+                }
+        }
+        const elasticSearchClientArgs: ElasticClientArgs = {
+            client: new Client(elasticSearchClientOptions),
+            indexName: indexName,
+            vectorSearchOptions: vectorSearchOptions
+        }
+
+        const flattenDocs = docs && docs.length ? flatten(docs) : []
+        const finalDocs = []
+        for (let i = 0; i < flattenDocs.length; i += 1) {
+            finalDocs.push(new Document(flattenDocs[i]))
+        }
+
+        const vectorStore = await ElasticVectorSearch.fromDocuments(finalDocs, embeddings, elasticSearchClientArgs)
+
+        if (output === 'retriever') {
+            return vectorStore.asRetriever(k)
+        } else if (output === 'vectorStore') {
+            ;(vectorStore as any).k = k
+            return vectorStore
+        }
+        return vectorStore
+    }
+}
+
+module.exports = { nodeClass: ElasicsearchUpsert_VectorStores }
diff --git a/packages/components/nodes/vectorstores/Elasticsearch/elasticsearch.png b/packages/components/nodes/vectorstores/Elasticsearch/elasticsearch.png
new file mode 100644
index 0000000000000000000000000000000000000000..fdb6686369bf0186319e24f2dd96ebee1ee6a54b
GIT binary patch
literal 3719
zcmV;24tVj2P)<h;3K|Lk000e1NJLTq007|t007|#0{{R332h=s0003pP)t-s|Ns9y
zzOnSZ85d?%EJcs$M~s!mJ@dE#^t%A_w*Vlyt1-N;0AWu6UQ6jjivU$X=r)8SKa2n^
zg#cMaj>9SFK8O}Ki5ohKy5!%H#4**_<sv?e0568~_VoZMgniG(i@_iM`tbbg$^7uv
z_tJZl*3VSLxW(w?Xvx3*_T~85lK0MQ{O{BH<FWL=Cj02X_Q+KE*W>Slt?-k#^|sCG
zRFnSp`Qqv4ufx5-&&$~G?dI{<^~6K@-lObfoA}m;{Ot1i*yQ%Y)$Hx_-r(vsO_OeW
zt6OWLqN&GbkcEnrx+!g5)85;{$J&;#sYH8jr`^}Q+K;8qV$0&G-t5W7-=OoP!uGt;
z@QSkb%U|nan(cF><`Q|dy3u!ouwihefT5X3TA5abd7R2lwAO&)?$0N9@5%rG4G>90
zK~#90?VWpD6FC&dX(?^#g%%2h0)-+_xnD#D5fm>VcLdR0*9AfJ`+tR<UM87jl9@A;
zleFy9_pg1{=}do~<UJ=RncCIWQAZth)KN$6uc`U@>FMeDsaAgv4VqtFSlme_s8-fy
zcAxxTNS7AY5{YCo8D(Vsk3=Fmv;62OR~Kt^QJE~vm4)e6f64G#n(?9|l!=wy#~I1x
zm4rWD6S8;l@da`h5IrW4J5O5ul>;dtUtNiv#}miZXmGwd<M+ogAE(w5q4=V;vAWyp
zZx7He*k6QU+?j7RfS<w#&}MA!FfN5~7#HIAR#SLrw^y0ZQgWqj4QEDmgxX3hwHigN
z@9HDeR_|)7VMJQP1GEsATaDw^4kn<5*mjnR8o@fE(I~#$d$+#6{%%ix#UeJK!MV8i
z{8?Y#)|-v@!i(NwxhR8Zq7vWx){`F^>LKbI%IBZ0yDnSS0ca&g?RjzUS$?Rmhu2WP
z=P&lUd6YyiAl%UmxRD=n;WZ+efAP+Fp&q^(;*3td^&tsggYj>Ar3RO=mC)!m8+rHm
zbSl4LU-?AOBy1vTn~Fd3o_rP|G1XmtA+*M+u7$=w9wW3#xMyFhagau}jZ=TXo;<l&
zC!eLH4!|G&fPHW5i>Gu>!y4fNKHxXMIcRB!7|oJqLJel~4glDbCrMwc(bR;W6km9!
zi6$XQ`Ff3}re5SAK8J+RVKghc0i(&cxP&0UzJJK|x~HL~1S;S+e;@5}8qKr;!X)Z?
z9|Y(ynq}?NuqH&n_n<-QFq#<?gqYpMXTTVe59}BKL`@j55SlfbD5lqdvb|}esS%=W
z5I=8Eqd`oGe*+1L(Y!T7i07tnAQhW&qp_60+Boe25%#?xA22fwtznAQ0S6ilX+zB$
zBgC4BK*Hzt^{b|)VHB%%8nmKWFh+=#5rKr{dh{J9IU$nOfL6Z_qcH}Emh*o=2wPw@
zcxJi*$@r}>8nl+%U!X2j3ycPBd9eo~d}&{fc>2PZ5Uai4=RXl!Vl>g{8WQ1#Xit=z
zhR^Jab+@}2VWs?+?>AoLy(y{DXx&%iwSq)gr(FN`#)C3X_VKm1yLA!Pwf^l#J}=w`
zrbeT`-R+Q;J^gp}gC<-s`+CTq8|2eOSkoRqF<WFbNh~iFikX<yXteLEF@%dc3G1>a
zW{ZqQ@4*uyYhn)lON_=yyPFVV)TQN{Gr(1Q<iGUqt2qhP*A}A(6UE}h{qflc=qF50
zL-nnTv-|P>n5F));z0lH%}2mT47Iy$LbZ+H(I#J0W5x0DpNFG9A2QVLwh5I0ZwH)w
z4fPL?4|oq*qehcZNfu|tF&CdrSsd6uzL8KGT>eHAD!Jl#K*C3d?C^nT$QmqZl#aE3
zDtZHwV17D0E3+RpHcVAce3$oSBUCbEofC6P)6rmF35E0C<(S)IJSnXqTRbEmH?n$4
zMiufzP{4);$mdP8OV!SL?}q`54bvd%Z|buxwsP6U;HjU(VE|*XBDsV9bBRorD4*o+
zHUj`2LR?fxcth@?c?+1$5}TP!cP4iPW*nUe322GjWL0Yr*(`Bd=&mV~y8{x2nuE<C
z(p8qu6%?B#wsPG~WwWm!fX5*xVNLz_<Tm=s?_`wOF+3-LmU2h_fFBjcXe>5PgQ$y;
z%MwS0Zl|&*zOMR7VUSuu{6ub~saeCd)-JN$oC>GjfTKR;wx0%QC9cJ5_$-m(0j-hh
zJsH(PXa(_r+)BqHs+Vay)v|;@o`@e6Ll}!qkP2MGB8q)M_Ou}4^Fh1?uP9_1yCpY6
zUqrD@r>&d>&_a|a|8Z47XAqI!uS_Q8S%Oza=BQ_3>otdlkIC&&7g5|#MjXk8XoK)V
zLU^F)Gj)xV+fDc4A^K(sR@rlHgOpiB%`ajx0<r{TDvlxy(wK7*Wf|&QrQhAL1<Dev
zgyZy3p(Ldm+h@CsVSfCI2ZUt_MujZ_A<8ryzIWYsc||QXg0lppjDT<-0kHT}G&)vy
z!VlIUS;DETAc)EZjb)^_?%UQi{XHaOpe*4N#1HCLI&kc|E7AN+7{-8E!l{C*cpFtF
zoVeld$!KaJu<5WHFMurJjG@a;l`EPFW~+Ea1i2dxWIBo9S;8S#PE;ivXLe~OQ|Vbz
zb+`)u<x4(GKt0BmtFTcc>>p<Khv(9fNIEqsVq97SGO`Dpp;^KqewZo|_Q#k!U$p=u
zHO9GZ0VAlI3!5bz%5i_eDn%Qc=gW}>V4bo0RKVz8&4terRJ@yh3{P16<h;=~;j1eZ
z5u1*uu_|N<Dt;n{&xb`O&u^z9oYG>`@u=ohvIJFzZ=Z&(X~vm6KjH$6NKNO7q?+Th
zL^d3tCGL44>(YTk=S@C~0Oh7LEixT8OS}r<F-_&Tr-fTvu!YR51kl5DT-BU%mLMu{
zIX~3;aDdD6Qb4)saMc`}CGNtQC?Vt;;33jQf6<lay_HUX7BF@#lWGnlOGpWWTTOqz
zD9`&aro`1;!*sZfmvC8v67tK7p8+T0*uL|-I`X_9<D|%RYSkQ0mXH(P1RJJYAp_)V
zFwTigXMT}+=XFypfZd{vs6d;C#RKO@-Fe<yl1j%Rg)AW_JP5FF!8R<O1_X>qO~;l>
zmLP<#u4#Z}KHIR6<#``QvFX&MlqGEO8Q;=96cC)}{TPL&qnagb2|9{rz#y^>3!gmi
z!zile6tjd4_)GA2cMnP!Y{SAo&wDFX<xFQ&HA~opqWu}jMn2cD0Ofg~Kt>qT*+Lnj
z*7{Jm8SHPTY{LSa=Y1F%(>cK*<hvM6`DOhvu3>>Ci;>**6;2Q(`=n>y<8+v9SSWam
zODSqO7Z`xDZPKn|uPCyGOo%-1r7F2GYzNIp@&gO^@~H@DSSZL5#9s(*D%;GrBs~N@
z0cu!;7Y*X3dd()<UbFRKSd55N$jFtVD~==OAZF!&GDNPBLCF%l@Pa7ynI>Y)H5UUn
zEW#*Fs~goIdWNv3d(KhAG%Ui1JfW->M6X%c6#L#&$c9B=Io6DKN<lpF1+-HCFdeCE
z!(v+(Ukwo_h&i8wv`yVkG!2J6VS!r6xCm(w&%-aFvUecBuC9-bcSzZt7wkOGDMg<Y
zy0FPW#%NP#_v2$24GXx^v;Yqs#6qCAghmgTLPkTL=aiyfFC-yw5{jkLumH*t#H~<>
zIsZf<m!3S&$q~d%h^@n%cQNBFZF!zidXhYF^ZZw#5R6hKg8K42rxeqD+uS<p&VebF
z01fe3L7a0v`_<h^4iL;tR>5Sn=6OyjQJc7Y>N{k!7Xe=;bqcv*;VpL%ZJll%Wn`2Y
zC6DPc3Ftvs)8>{nWZjg7iy&{xx@2;UGR3Jx<yYqy%)~5ogI=$cYs8q%)CooVr*55|
zA2lXsrf{OjXH}y{358B0mNjI%3wJ7IT2p$uGEzeF+{-`e-yNnolYkLoG_Nazm50k~
z6aPxCQ9xRIUzg(Xc&Sv0|Nin?Goe{4B^|fR8ezOnIXHNwmC`67HJYoqLl$M_>Z`6H
zYnG5ab}-JXq-$atB_!Jy^KQbpriocLT9uOZ;kB4h%fvKFNNTpPB`G19m{^b|2^~h`
zO<1S={TK<!B2PX@sRoe<jTUY8L#HYbLZo3@HOB`s8pkmGf<tJqE(IG+lkgIU(BRsF
zE)IV~2;)~cga+4#)M!9}@dq5jrv_taKNDm$)-c5(G}xyhi^D$<qrpoT#t7-+@C$5|
zVi6iMQ@S{W8BIKnL^x||BGQ@-#)svl=B$P;4uM94RhSx^h-7j28fG-5uULeJCSq`-
zxyB+il=A}{O$Du^VQ868jphLah*i?mWM-HKHX5X*MNNh_XY9uTjRtR!cA0Tgx;TUx
zO$jR-)@35;)5YN>h_Hg%Q#9c<ba4nW8r%>XSq9U^A;@SdSpQgVQwEk$WN`>G8mtZm
zmG(nl^oYVWWEvvo!lty(L|U^!j0QDRG^N{4snJ{k0JZFZYbK;MoBuSdpq1Q=OdWxi
z?5pVq)NM9oap>Fb9@KV7L#x@)isp6P(`iZ%8jok_9H&*sb1vm6zf?h6*9)buvPm&#
zCtqJl_P8roy3g0>+J{Q<r`i5oRa|4WRQav@WR13|m=YXFocj7u87wt0OZ83dFA8U5
z;nGr{I?k`Nkk?;-Up>Sh9<DC6pNpc>aA4T`(5!i(8)~}h|4_3+Mr=PoWW;tGq75O=
z#r6b5bX3O=Vzil%-V#dr{UlZ^h48LkIs!kn-U=!EO-hpI+ehW93S+vhN2sllROs0#
z{@K>{Q#s^pOjX-^20FDp7vdN9e1BE2y<{;48ms=ln%I8Yo&+;Wes1>>=yMeSnNFo=
zAJ6=Asa%7UQ>Gs|UPI-lv(=P^ndadvy2+78k7sLpWOlNeP7y`sW<NiD9Jw!;%H{HQ
lIq1LD+fhdyb<|ON>OYVv9CTA0p*R2l002ovPDHLkV1i_tAvOR2

literal 0
HcmV?d00001

diff --git a/packages/components/package.json b/packages/components/package.json
index 93609106..3fdd8923 100644
--- a/packages/components/package.json
+++ b/packages/components/package.json
@@ -19,6 +19,7 @@
         "@aws-sdk/client-dynamodb": "^3.360.0",
         "@dqbd/tiktoken": "^1.0.7",
         "@getzep/zep-js": "^0.6.3",
+        "@elastic/elasticsearch": "^8.9.0",
         "@google-ai/generativelanguage": "^0.2.1",
         "@huggingface/inference": "^2.6.1",
         "@notionhq/client": "^2.2.8",

From 57760dc633a1700316e7c3fc3427d82e89e19eec Mon Sep 17 00:00:00 2001
From: vinodkiran <vinodkiran@usa.net>
Date: Tue, 10 Oct 2023 20:03:21 +0530
Subject: [PATCH 2/3] Updates to Elasticsearch VectoreStore Functionality.

---
 .../ElectricsearchUserPassword.credential.ts  |   9 +-
 .../Elasticsearch/ElasticSearchBase.ts        | 193 ++++++++++++++++++
 .../Elasticsearch/Elasticsearch_Existing.ts   | 112 ++--------
 .../Elasticsearch/Elasticsearch_Upsert.ts     | 168 +++------------
 4 files changed, 245 insertions(+), 237 deletions(-)
 create mode 100644 packages/components/nodes/vectorstores/Elasticsearch/ElasticSearchBase.ts

diff --git a/packages/components/credentials/ElectricsearchUserPassword.credential.ts b/packages/components/credentials/ElectricsearchUserPassword.credential.ts
index 2dd88937..6c47f7b1 100644
--- a/packages/components/credentials/ElectricsearchUserPassword.credential.ts
+++ b/packages/components/credentials/ElectricsearchUserPassword.credential.ts
@@ -14,14 +14,19 @@ class ElasticSearchUserPassword implements INodeCredential {
         this.description =
             'Refer to <a target="_blank" href="https://www.elastic.co/guide/en/kibana/current/tutorial-secure-access-to-kibana.html">official guide</a> on how to get User Password from ElasticSearch'
         this.inputs = [
+            {
+                label: 'Cloud ID',
+                name: 'cloudId',
+                type: 'string'
+            },
             {
                 label: 'ElasticSearch User',
-                name: 'elasticSearchUser',
+                name: 'username',
                 type: 'string'
             },
             {
                 label: 'ElasticSearch Password',
-                name: 'elasticSearchPassword',
+                name: 'password',
                 type: 'password'
             }
         ]
diff --git a/packages/components/nodes/vectorstores/Elasticsearch/ElasticSearchBase.ts b/packages/components/nodes/vectorstores/Elasticsearch/ElasticSearchBase.ts
new file mode 100644
index 00000000..59294b7e
--- /dev/null
+++ b/packages/components/nodes/vectorstores/Elasticsearch/ElasticSearchBase.ts
@@ -0,0 +1,193 @@
+import {
+    getBaseClasses,
+    getCredentialData,
+    getCredentialParam,
+    ICommonObject,
+    INodeData,
+    INodeOutputsValue,
+    INodeParams
+} from '../../../src'
+import { Client, ClientOptions } from '@elastic/elasticsearch'
+import { ElasticClientArgs, ElasticVectorSearch } from 'langchain/vectorstores/elasticsearch'
+import { Embeddings } from 'langchain/embeddings/base'
+import { VectorStore } from 'langchain/vectorstores/base'
+import { Document } from 'langchain/document'
+
+export abstract class ElasticSearchBase {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    credential: INodeParams
+    outputs: INodeOutputsValue[]
+
+    protected constructor() {
+        this.type = 'Elasticsearch'
+        this.icon = 'elasticsearch.png'
+        this.category = 'Vector Stores'
+        this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['elasticsearchApi', 'elasticSearchUserPassword']
+        }
+        this.inputs = [
+            {
+                label: 'Embeddings',
+                name: 'embeddings',
+                type: 'Embeddings'
+            },
+            {
+                label: 'Index Name',
+                name: 'indexName',
+                placeholder: '<INDEX_NAME>',
+                type: 'string'
+            },
+            {
+                label: 'Top K',
+                name: 'topK',
+                description: 'Number of top results to fetch. Default to 4',
+                placeholder: '4',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Similarity',
+                name: 'similarity',
+                description: 'Similarity measure used in Elasticsearch.',
+                type: 'options',
+                default: 'l2_norm',
+                options: [
+                    {
+                        label: 'l2_norm',
+                        name: 'l2_norm'
+                    },
+                    {
+                        label: 'dot_product',
+                        name: 'dot_product'
+                    },
+                    {
+                        label: 'cosine',
+                        name: 'cosine'
+                    }
+                ],
+                additionalParams: true,
+                optional: true
+            }
+        ]
+        this.outputs = [
+            {
+                label: 'Elasticsearch Retriever',
+                name: 'retriever',
+                baseClasses: this.baseClasses
+            },
+            {
+                label: 'Elasticsearch Vector Store',
+                name: 'vectorStore',
+                baseClasses: [this.type, ...getBaseClasses(ElasticVectorSearch)]
+            }
+        ]
+    }
+
+    abstract constructVectorStore(
+        embeddings: Embeddings,
+        elasticSearchClientArgs: ElasticClientArgs,
+        docs: Document<Record<string, any>>[] | undefined
+    ): Promise<VectorStore>
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject, docs: Document<Record<string, any>>[] | undefined): Promise<any> {
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const endPoint = getCredentialParam('endpoint', credentialData, nodeData)
+        const cloudId = getCredentialParam('cloudId', credentialData, nodeData)
+        const indexName = nodeData.inputs?.indexName as string
+        const embeddings = nodeData.inputs?.embeddings as Embeddings
+        const topK = nodeData.inputs?.topK as string
+        const similarityMeasure = nodeData.inputs?.similarityMeasure as string
+        const k = topK ? parseFloat(topK) : 4
+        const output = nodeData.outputs?.output as string
+
+        const elasticSearchClientArgs = this.prepareClientArgs(endPoint, cloudId, credentialData, nodeData, similarityMeasure, indexName)
+
+        const vectorStore = await this.constructVectorStore(embeddings, elasticSearchClientArgs, docs)
+
+        if (output === 'retriever') {
+            return vectorStore.asRetriever(k)
+        } else if (output === 'vectorStore') {
+            ;(vectorStore as any).k = k
+            return vectorStore
+        }
+        return vectorStore
+    }
+
+    protected prepareConnectionOptions(
+        endPoint: string | undefined,
+        cloudId: string | undefined,
+        credentialData: ICommonObject,
+        nodeData: INodeData
+    ) {
+        let elasticSearchClientOptions: ClientOptions = {}
+        if (endPoint) {
+            let apiKey = getCredentialParam('apiKey', credentialData, nodeData)
+            elasticSearchClientOptions = {
+                node: endPoint,
+                auth: {
+                    apiKey: apiKey
+                }
+            }
+        } else if (cloudId) {
+            let username = getCredentialParam('username', credentialData, nodeData)
+            let password = getCredentialParam('password', credentialData, nodeData)
+            elasticSearchClientOptions = {
+                cloud: {
+                    id: cloudId
+                },
+                auth: {
+                    username: username,
+                    password: password
+                }
+            }
+        }
+        return elasticSearchClientOptions
+    }
+
+    protected prepareClientArgs(
+        endPoint: string | undefined,
+        cloudId: string | undefined,
+        credentialData: ICommonObject,
+        nodeData: INodeData,
+        similarityMeasure: string,
+        indexName: string
+    ) {
+        let elasticSearchClientOptions = this.prepareConnectionOptions(endPoint, cloudId, credentialData, nodeData)
+        let vectorSearchOptions = {}
+        switch (similarityMeasure) {
+            case 'dot_product':
+                vectorSearchOptions = {
+                    similarity: 'dot_product'
+                }
+                break
+            case 'cosine':
+                vectorSearchOptions = {
+                    similarity: 'cosine'
+                }
+                break
+            default:
+                vectorSearchOptions = {
+                    similarity: 'l2_norm'
+                }
+        }
+        const elasticSearchClientArgs: ElasticClientArgs = {
+            client: new Client(elasticSearchClientOptions),
+            indexName: indexName,
+            vectorSearchOptions: vectorSearchOptions
+        }
+        return elasticSearchClientArgs
+    }
+}
diff --git a/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Existing.ts b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Existing.ts
index 6e785c85..94e45d74 100644
--- a/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Existing.ts
+++ b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Existing.ts
@@ -1,110 +1,30 @@
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData } from '../../../src/Interface'
 import { Embeddings } from 'langchain/embeddings/base'
-import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src'
 
-import { Client, ClientOptions } from '@elastic/elasticsearch'
 import { ElasticClientArgs, ElasticVectorSearch } from 'langchain/vectorstores/elasticsearch'
+import { ElasticSearchBase } from './ElasticSearchBase'
+import { VectorStore } from 'langchain/vectorstores/base'
+import { Document } from 'langchain/document'
 
-class ElasicsearchExisting_VectorStores implements INode {
-    label: string
-    name: string
-    version: number
-    description: string
-    type: string
-    icon: string
-    category: string
-    baseClasses: string[]
-    inputs: INodeParams[]
-    credential: INodeParams
-    outputs: INodeOutputsValue[]
-
+class ElasicsearchExisting_VectorStores extends ElasticSearchBase implements INode {
     constructor() {
+        super()
         this.label = 'Elasticsearch Load Existing Index'
         this.name = 'ElasticsearchIndex'
         this.version = 1.0
-        this.type = 'Elasticsearch'
-        this.icon = 'elasticsearch.png'
-        this.category = 'Vector Stores'
-        this.description = 'Load existing index from Elasticsearch   (i.e: Document has been upserted)'
-        this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
-        this.credential = {
-            label: 'Connect Credential',
-            name: 'credential',
-            type: 'credential',
-            credentialNames: ['elasticsearchApi', 'elasticSearchUserPassword']
-        }
-        this.inputs = [
-            {
-                label: 'Embeddings',
-                name: 'embeddings',
-                type: 'Embeddings'
-            },
-            {
-                label: 'Index Name',
-                name: 'indexName',
-                placeholder: '<INDEX_NAME>',
-                type: 'string'
-            },
-            {
-                label: 'Top K',
-                name: 'topK',
-                description: 'Number of top results to fetch. Default to 4',
-                placeholder: '4',
-                type: 'number',
-                additionalParams: true,
-                optional: true
-            }
-        ]
-        this.outputs = [
-            {
-                label: 'Elasticsearch Retriever',
-                name: 'retriever',
-                baseClasses: this.baseClasses
-            },
-            {
-                label: 'Elasticsearch Vector Store',
-                name: 'vectorStore',
-                baseClasses: [this.type, ...getBaseClasses(ElasticVectorSearch)]
-            }
-        ]
+        this.description = 'Load existing index from Elasticsearch (i.e: Document has been upserted)'
+    }
+
+    async constructVectorStore(
+        embeddings: Embeddings,
+        elasticSearchClientArgs: ElasticClientArgs,
+        docs: Document<Record<string, any>>[] | undefined
+    ): Promise<VectorStore> {
+        return await ElasticVectorSearch.fromExistingIndex(embeddings, elasticSearchClientArgs)
     }
 
     async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
-        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
-        const endPoint = getCredentialParam('endpoint', credentialData, nodeData)
-        const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
-        const indexName = nodeData.inputs?.indexName as string
-        const embeddings = nodeData.inputs?.embeddings as Embeddings
-        const topK = nodeData.inputs?.topK as string
-
-        const k = topK ? parseFloat(topK) : 4
-        const output = nodeData.outputs?.output as string
-
-        // eslint-disable-next-line no-console
-        console.log('EndPoint:: ' + endPoint + ', APIKey:: ' + apiKey + ', Index:: ' + indexName)
-
-        const elasticSearchClientOptions: ClientOptions = {
-            node: endPoint,
-            auth: {
-                apiKey: apiKey
-            }
-        }
-
-        const elasticSearchClientArgs: ElasticClientArgs = {
-            client: new Client(elasticSearchClientOptions),
-            indexName: indexName
-        }
-
-        const vectorStore = await ElasticVectorSearch.fromExistingIndex(embeddings, elasticSearchClientArgs)
-        // eslint-disable-next-line no-console
-        console.log('vectorStore ::' + vectorStore._vectorstoreType())
-        if (output === 'retriever') {
-            return vectorStore.asRetriever(k)
-        } else if (output === 'vectorStore') {
-            ;(vectorStore as any).k = k
-            return vectorStore
-        }
-        return vectorStore
+        return super.init(nodeData, _, options, undefined)
     }
 }
 
diff --git a/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts
index 5a0065d5..d4b79a5d 100644
--- a/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts
+++ b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts
@@ -1,148 +1,39 @@
-import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData } from '../../../src/Interface'
 import { Embeddings } from 'langchain/embeddings/base'
 import { Document } from 'langchain/document'
-import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src'
 
-import { Client, ClientOptions } from '@elastic/elasticsearch'
 import { ElasticClientArgs, ElasticVectorSearch } from 'langchain/vectorstores/elasticsearch'
 import { flatten } from 'lodash'
+import { ElasticSearchBase } from './ElasticSearchBase'
+import { VectorStore } from 'langchain/vectorstores/base'
 
-class ElasicsearchUpsert_VectorStores implements INode {
-    label: string
-    name: string
-    version: number
-    description: string
-    type: string
-    icon: string
-    category: string
-    baseClasses: string[]
-    inputs: INodeParams[]
-    credential: INodeParams
-    outputs: INodeOutputsValue[]
-
+class ElasicsearchUpsert_VectorStores extends ElasticSearchBase implements INode {
     constructor() {
+        super()
         this.label = 'Elasticsearch Upsert Document'
         this.name = 'ElasticsearchUpsert'
         this.version = 1.0
-        this.type = 'Elasticsearch'
-        this.icon = 'elasticsearch.png'
-        this.category = 'Vector Stores'
         this.description = 'Upsert documents to Elasticsearch'
-        this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
-        this.credential = {
-            label: 'Connect Credential',
-            name: 'credential',
-            type: 'credential',
-            credentialNames: ['elasticsearchApi', 'elasticSearchUserPassword']
-        }
-        this.inputs = [
-            {
-                label: 'Document',
-                name: 'document',
-                type: 'Document',
-                list: true
-            },
-            {
-                label: 'Embeddings',
-                name: 'embeddings',
-                type: 'Embeddings'
-            },
-            {
-                label: 'Index Name',
-                name: 'indexName',
-                placeholder: '<INDEX_NAME>',
-                type: 'string'
-            },
-            {
-                label: 'Top K',
-                name: 'topK',
-                description: 'Number of top results to fetch. Default to 4',
-                placeholder: '4',
-                type: 'number',
-                additionalParams: true,
-                optional: true
-            },
-            {
-                label: 'Similarity',
-                name: 'similarity',
-                description: 'Similarity measure used in Elasticsearch.',
-                type: 'options',
-                default: 'l2_norm',
-                options: [
-                    {
-                        label: 'l2_norm',
-                        name: 'l2_norm'
-                    },
-                    {
-                        label: 'dot_product',
-                        name: 'dot_product'
-                    },
-                    {
-                        label: 'cosine',
-                        name: 'cosine'
-                    }
-                ],
-                additionalParams: true,
-                optional: true
-            }
-        ]
-        this.outputs = [
-            {
-                label: 'Elasticsearch Retriever',
-                name: 'retriever',
-                baseClasses: this.baseClasses
-            },
-            {
-                label: 'Elasticsearch Vector Store',
-                name: 'vectorStore',
-                baseClasses: [this.type, ...getBaseClasses(ElasticVectorSearch)]
-            }
-        ]
+        this.inputs.unshift({
+            label: 'Document',
+            name: 'document',
+            type: 'Document',
+            list: true
+        })
+    }
+
+    async constructVectorStore(
+        embeddings: Embeddings,
+        elasticSearchClientArgs: ElasticClientArgs,
+        docs: Document<Record<string, any>>[]
+    ): Promise<VectorStore> {
+        const vectorStore = new ElasticVectorSearch(embeddings, elasticSearchClientArgs)
+        await vectorStore.addDocuments(docs)
+        return vectorStore
     }
 
     async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
-        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
-        const endPoint = getCredentialParam('endpoint', credentialData, nodeData)
-        const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
         const docs = nodeData.inputs?.document as Document[]
-        const indexName = nodeData.inputs?.indexName as string
-        const embeddings = nodeData.inputs?.embeddings as Embeddings
-        const topK = nodeData.inputs?.topK as string
-        const k = topK ? parseFloat(topK) : 4
-        const output = nodeData.outputs?.output as string
-        const similarityMeasure = nodeData.inputs?.similarityMeasure as string
-
-        // eslint-disable-next-line no-console
-        console.log('EndPoint:: ' + endPoint + ', APIKey:: ' + apiKey + ', Index:: ' + indexName)
-
-        const elasticSearchClientOptions: ClientOptions = {
-            node: endPoint,
-            auth: {
-                apiKey: apiKey
-            }
-        }
-        let vectorSearchOptions = {}
-        switch (similarityMeasure) {
-            case 'dot_product':
-                vectorSearchOptions = {
-                    similarity: 'dot_product'
-                }
-                break
-            case 'cosine':
-                vectorSearchOptions = {
-                    similarity: 'cosine'
-                }
-                break
-            default:
-                vectorSearchOptions = {
-                    similarity: 'l2_norm'
-                }
-        }
-        const elasticSearchClientArgs: ElasticClientArgs = {
-            client: new Client(elasticSearchClientOptions),
-            indexName: indexName,
-            vectorSearchOptions: vectorSearchOptions
-        }
 
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
@@ -150,15 +41,14 @@ class ElasicsearchUpsert_VectorStores implements INode {
             finalDocs.push(new Document(flattenDocs[i]))
         }
 
-        const vectorStore = await ElasticVectorSearch.fromDocuments(finalDocs, embeddings, elasticSearchClientArgs)
-
-        if (output === 'retriever') {
-            return vectorStore.asRetriever(k)
-        } else if (output === 'vectorStore') {
-            ;(vectorStore as any).k = k
-            return vectorStore
-        }
-        return vectorStore
+        // The following code is a workaround for a bug (Langchain Issue #1589) in the underlying library.
+        // Store does not support object in metadata and fail silently
+        finalDocs.forEach((d) => {
+            delete d.metadata.pdf
+            delete d.metadata.loc
+        })
+        // end of workaround
+        return super.init(nodeData, _, options, flattenDocs)
     }
 }
 

From 8bca7b3c04ad300877822e3d224763f45ba3d6dc Mon Sep 17 00:00:00 2001
From: Henry Heng <henryheng@flowiseai.com>
Date: Thu, 12 Oct 2023 12:40:43 +0100
Subject: [PATCH 3/3] Update ElasticsearchAPI.credential.ts

---
 packages/components/credentials/ElasticsearchAPI.credential.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/components/credentials/ElasticsearchAPI.credential.ts b/packages/components/credentials/ElasticsearchAPI.credential.ts
index e377243d..fbba76f4 100644
--- a/packages/components/credentials/ElasticsearchAPI.credential.ts
+++ b/packages/components/credentials/ElasticsearchAPI.credential.ts
@@ -20,7 +20,7 @@ class ElectricsearchAPI implements INodeCredential {
                 type: 'string'
             },
             {
-                label: 'Elasticsearch API ID',
+                label: 'Elasticsearch API Key',
                 name: 'apiKey',
                 type: 'password'
             }