Merge pull request #645 from vectara/main

Feature/Add Vectara Vector Store

packages/components/credentials/VectaraApi.credential.ts

@@ -0,0 +1,34 @@
+import { INodeParams, INodeCredential } from '../src/Interface'
+
+class VectaraAPI implements INodeCredential {
+    label: string
+    name: string
+    version: number
+    description: string
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'Vectara API'
+        this.name = 'vectaraApi'
+        this.version = 1.0
+        this.inputs = [
+            {
+                label: 'Vectara Customer ID',
+                name: 'customerID',
+                type: 'string'
+            },
+            {
+                label: 'Vectara Corpus ID',
+                name: 'corpusID',
+                type: 'string'
+            },
+            {
+                label: 'Vectara API Key',
+                name: 'apiKey',
+                type: 'password'
+            }
+        ]
+    }
+}
+
+module.exports = { credClass: VectaraAPI }

packages/components/nodes/vectorstores/Vectara_Existing/Vectara_Existing.ts

@@ -0,0 +1,111 @@
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { VectaraStore, VectaraLibArgs, VectaraFilter } from 'langchain/vectorstores/vectara'
+
+class VectaraExisting_VectorStores implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    credential: INodeParams
+    outputs: INodeOutputsValue[]
+
+    constructor() {
+        this.label = 'Vectara Load Existing Index'
+        this.name = 'vectaraExistingIndex'
+        this.version = 1.0
+        this.type = 'Vectara'
+        this.icon = 'vectara.png'
+        this.category = 'Vector Stores'
+        this.description = 'Load existing index from Vectara (i.e: Document has been upserted)'
+        this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['vectaraApi']
+        }
+        this.inputs = [
+            {
+                label: 'Vectara Metadata Filter',
+                name: 'filter',
+                type: 'json',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Lambda',
+                name: 'lambda',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Top K',
+                name: 'topK',
+                description: 'Number of top results to fetch. Defaults to 4',
+                placeholder: '4',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            }
+        ]
+        this.outputs = [
+            {
+                label: 'Vectara Retriever',
+                name: 'retriever',
+                baseClasses: this.baseClasses
+            },
+            {
+                label: 'Vectara Vector Store',
+                name: 'vectorStore',
+                baseClasses: [this.type, ...getBaseClasses(VectaraStore)]
+            }
+        ]
+    }
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
+        const customerId = getCredentialParam('customerID', credentialData, nodeData)
+        const corpusId = getCredentialParam('corpusID', credentialData, nodeData)
+
+        const vectaraMetadatafilter = nodeData.inputs?.filter as VectaraFilter
+        const lambda = nodeData.inputs?.lambda as number
+        const output = nodeData.outputs?.output as string
+        const topK = nodeData.inputs?.topK as string
+        const k = topK ? parseInt(topK, 10) : 4
+
+        const vectaraArgs: VectaraLibArgs = {
+            apiKey: apiKey,
+            customerId: customerId,
+            corpusId: corpusId
+        }
+
+        const vectaraFilter: VectaraFilter = {}
+
+        if (vectaraMetadatafilter) {
+            const metadatafilter = typeof vectaraMetadatafilter === 'object' ? vectaraMetadatafilter : JSON.parse(vectaraMetadatafilter)
+            vectaraFilter.filter = metadatafilter
+        }
+
+        if (lambda) vectaraFilter.lambda = lambda
+
+        const vectorStore = new VectaraStore(vectaraArgs)
+
+        if (output === 'retriever') {
+            const retriever = vectorStore.asRetriever(k, vectaraFilter)
+            return retriever
+        } else if (output === 'vectorStore') {
+            ;(vectorStore as any).k = k
+            return vectorStore
+        }
+        return vectorStore
+    }
+}
+
+module.exports = { nodeClass: VectaraExisting_VectorStores }

packages/components/nodes/vectorstores/Vectara_Upsert/Vectara_Upsert.ts

@@ -0,0 +1,128 @@
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { Embeddings } from 'langchain/embeddings/base'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { VectaraStore, VectaraLibArgs, VectaraFilter } from 'langchain/vectorstores/vectara'
+import { Document } from 'langchain/document'
+import { flatten } from 'lodash'
+
+class VectaraExisting_VectorStores implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    credential: INodeParams
+    outputs: INodeOutputsValue[]
+
+    constructor() {
+        this.label = 'Vectara Upsert Document'
+        this.name = 'vectaraExisting'
+        this.version = 1.0
+        this.type = 'Vectara'
+        this.icon = 'vectara.png'
+        this.category = 'Vector Stores'
+        this.description = 'Upsert documents to Vectara'
+        this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['vectaraApi']
+        }
+        this.inputs = [
+            {
+                label: 'Document',
+                name: 'document',
+                type: 'Document',
+                list: true
+            },
+            {
+                label: 'Filter',
+                name: 'filter',
+                type: 'json',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Lambda',
+                name: 'lambda',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Top K',
+                name: 'topK',
+                description: 'Number of top results to fetch. Defaults to 4',
+                placeholder: '4',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            }
+        ]
+        this.outputs = [
+            {
+                label: 'Vectara Retriever',
+                name: 'retriever',
+                baseClasses: this.baseClasses
+            },
+            {
+                label: 'Vectara Vector Store',
+                name: 'vectorStore',
+                baseClasses: [this.type, ...getBaseClasses(VectaraStore)]
+            }
+        ]
+    }
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
+        const customerId = getCredentialParam('customerID', credentialData, nodeData)
+        const corpusId = getCredentialParam('corpusID', credentialData, nodeData)
+
+        const docs = nodeData.inputs?.document as Document[]
+        const embeddings = {} as Embeddings
+        const vectaraMetadatafilter = nodeData.inputs?.filter as VectaraFilter
+        const lambda = nodeData.inputs?.lambda as number
+        const output = nodeData.outputs?.output as string
+        const topK = nodeData.inputs?.topK as string
+        const k = topK ? parseInt(topK, 10) : 4
+
+        const vectaraArgs: VectaraLibArgs = {
+            apiKey: apiKey,
+            customerId: customerId,
+            corpusId: corpusId
+        }
+
+        const vectaraFilter: VectaraFilter = {}
+
+        if (vectaraMetadatafilter) {
+            const metadatafilter = typeof vectaraMetadatafilter === 'object' ? vectaraMetadatafilter : JSON.parse(vectaraMetadatafilter)
+            vectaraFilter.filter = metadatafilter
+        }
+
+        if (lambda) vectaraFilter.lambda = lambda
+
+        const flattenDocs = docs && docs.length ? flatten(docs) : []
+        const finalDocs = []
+        for (let i = 0; i < flattenDocs.length; i += 1) {
+            finalDocs.push(new Document(flattenDocs[i]))
+        }
+
+        const vectorStore = await VectaraStore.fromDocuments(finalDocs, embeddings, vectaraArgs)
+
+        if (output === 'retriever') {
+            const retriever = vectorStore.asRetriever(k, vectaraFilter)
+            return retriever
+        } else if (output === 'vectorStore') {
+            ;(vectorStore as any).k = k
+            return vectorStore
+        }
+        return vectorStore
+    }
+}
+
+module.exports = { nodeClass: VectaraExisting_VectorStores }