From 6a8f7a314d95dbf1530a2321989074aa43aa443f Mon Sep 17 00:00:00 2001 From: Seif Date: Wed, 30 Aug 2023 11:11:28 -0700 Subject: [PATCH] Create Vectara_Upload.ts --- .../vectorstores/Vectara/Vectara_Upload.ts | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 packages/components/nodes/vectorstores/Vectara/Vectara_Upload.ts diff --git a/packages/components/nodes/vectorstores/Vectara/Vectara_Upload.ts b/packages/components/nodes/vectorstores/Vectara/Vectara_Upload.ts new file mode 100644 index 00000000..a7a2d799 --- /dev/null +++ b/packages/components/nodes/vectorstores/Vectara/Vectara_Upload.ts @@ -0,0 +1,161 @@ +import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { Embeddings } from 'langchain/embeddings/base' +import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { VectaraStore, VectaraLibArgs, VectaraFilter, VectaraContextConfig } from 'langchain/vectorstores/vectara' +import { Document } from 'langchain/document' +import { flatten } from 'lodash' + +class VectaraUpload_VectorStores implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + credential: INodeParams + outputs: INodeOutputsValue[] + + constructor() { + this.label = 'Vectara Upload File' + this.name = 'vectaraUpload' + this.version = 1.0 + this.type = 'Vectara' + this.icon = 'vectara.png' + this.category = 'Vector Stores' + this.description = 'Upload files to Vectara' + this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever'] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + credentialNames: ['vectaraApi'] + } + this.inputs = [ + { + label: 'File', + name: 'file', + type: 'file' + }, + { + label: 'Vectara Metadata Filter', + name: 'filter', + description: + 'Filter to apply to Vectara metadata. Refer to the documentation on how to use Vectara filters with Flowise.', + type: 'string', + additionalParams: true, + optional: true + }, + { + label: 'Sentences Before', + name: 'sentencesBefore', + description: 'Number of sentences to fetch before the matched sentence. Defaults to 2.', + type: 'number', + additionalParams: true, + optional: true + }, + { + label: 'Sentences After', + name: 'sentencesAfter', + description: 'Number of sentences to fetch after the matched sentence. Defaults to 2.', + type: 'number', + additionalParams: true, + optional: true + }, + { + label: 'Lambda', + name: 'lambda', + description: + 'Improves retrieval accuracy by adjusting the balance (from 0 to 1) between neural search and keyword-based search factors.', + type: 'number', + additionalParams: true, + optional: true + }, + { + label: 'Top K', + name: 'topK', + description: 'Number of top results to fetch. Defaults to 4', + placeholder: '4', + type: 'number', + additionalParams: true, + optional: true + } + ] + this.outputs = [ + { + label: 'Vectara Retriever', + name: 'retriever', + baseClasses: this.baseClasses + }, + { + label: 'Vectara Vector Store', + name: 'vectorStore', + baseClasses: [this.type, ...getBaseClasses(VectaraStore)] + } + ] + } + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const apiKey = getCredentialParam('apiKey', credentialData, nodeData) + const customerId = getCredentialParam('customerID', credentialData, nodeData) + const corpusId = getCredentialParam('corpusID', credentialData, nodeData) + + const fileBase64 = nodeData.inputs?.file + const vectaraMetadataFilter = nodeData.inputs?.filter as string + const sentencesBefore = nodeData.inputs?.sentencesBefore as number + const sentencesAfter = nodeData.inputs?.sentencesAfter as number + const lambda = nodeData.inputs?.lambda as number + const output = nodeData.outputs?.output as string + const topK = nodeData.inputs?.topK as string + const k = topK ? parseInt(topK, 10) : 4 + + const vectaraArgs: VectaraLibArgs = { + apiKey: apiKey, + customerId: customerId, + corpusId: corpusId + } + + const vectaraFilter: VectaraFilter = {} + if (vectaraMetadataFilter) vectaraFilter.filter = vectaraMetadataFilter + if (lambda) vectaraFilter.lambda = lambda + + const vectaraContextConfig: VectaraContextConfig = {} + if (sentencesBefore) vectaraContextConfig.sentencesBefore = sentencesBefore + if (sentencesAfter) vectaraContextConfig.sentencesAfter = sentencesAfter + vectaraFilter.contextConfig = vectaraContextConfig + + let files: string[] = [] + + if (fileBase64.startsWith('[') && fileBase64.endsWith(']')) { + files = JSON.parse(fileBase64) + } else { + files = [fileBase64] + } + + const blobs: Blob[] = [] + for (const file of files) { + const splitDataURI = file.split(',') + splitDataURI.pop() + const bf = Buffer.from(splitDataURI.pop() || '', 'base64') + const blob = new Blob([bf]) + blobs.push(blob) + } + + const vectorStore = new VectaraStore(vectaraArgs) + const res = await vectorStore.addFiles(blobs) + files = [] + + if (output === 'retriever') { + const retriever = vectorStore.asRetriever(k, vectaraFilter) + return retriever + } else if (output === 'vectorStore') { + ;(vectorStore as any).k = k + return vectorStore + } + return vectorStore + } +} + +module.exports = { nodeClass: VectaraUpload_VectorStores }