From f80547af60f9d5a54b2e916b83888abf4e016004 Mon Sep 17 00:00:00 2001 From: Seif Date: Tue, 15 Aug 2023 11:45:31 -0700 Subject: [PATCH 1/2] Add sentence config to Flowise --- .../Vectara_Existing/Vectara_Existing.ts | 23 ++++++++++++++++++- .../Vectara_Upsert/Vectara_Upsert.ts | 23 ++++++++++++++++++- packages/components/package.json | 2 +- 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/packages/components/nodes/vectorstores/Vectara_Existing/Vectara_Existing.ts b/packages/components/nodes/vectorstores/Vectara_Existing/Vectara_Existing.ts index f344338a..80fd0639 100644 --- a/packages/components/nodes/vectorstores/Vectara_Existing/Vectara_Existing.ts +++ b/packages/components/nodes/vectorstores/Vectara_Existing/Vectara_Existing.ts @@ -1,6 +1,6 @@ import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' -import { VectaraStore, VectaraLibArgs, VectaraFilter } from 'langchain/vectorstores/vectara' +import { VectaraStore, VectaraLibArgs, VectaraFilter, VectaraContextConfig } from 'langchain/vectorstores/vectara' class VectaraExisting_VectorStores implements INode { label: string @@ -40,6 +40,20 @@ class VectaraExisting_VectorStores implements INode { additionalParams: true, optional: true }, + { + label: 'Sentences Before', + name: 'sentencesBefore', + type: 'number', + additionalParams: true, + optional: true + }, + { + label: 'Sentences After', + name: 'sentencesAfter', + type: 'number', + additionalParams: true, + optional: true + }, { label: 'Lambda', name: 'lambda', @@ -77,6 +91,8 @@ class VectaraExisting_VectorStores implements INode { const corpusId = getCredentialParam('corpusID', credentialData, nodeData) const vectaraMetadataFilter = nodeData.inputs?.filter as string + const sentencesBefore = nodeData.inputs?.sentencesBefore as number + const sentencesAfter = nodeData.inputs?.sentencesAfter as number const lambda = nodeData.inputs?.lambda as number const output = nodeData.outputs?.output as string const topK = nodeData.inputs?.topK as string @@ -92,6 +108,11 @@ class VectaraExisting_VectorStores implements INode { if (vectaraMetadataFilter) vectaraFilter.filter = vectaraMetadataFilter if (lambda) vectaraFilter.lambda = lambda + const vectaraContextConfig: VectaraContextConfig = {} + if (sentencesBefore) vectaraContextConfig.sentencesBefore = sentencesBefore + if (sentencesAfter) vectaraContextConfig.sentencesAfter = sentencesAfter + vectaraFilter.contextConfig = vectaraContextConfig + const vectorStore = new VectaraStore(vectaraArgs) if (output === 'retriever') { diff --git a/packages/components/nodes/vectorstores/Vectara_Upsert/Vectara_Upsert.ts b/packages/components/nodes/vectorstores/Vectara_Upsert/Vectara_Upsert.ts index b2ee79e7..cda03240 100644 --- a/packages/components/nodes/vectorstores/Vectara_Upsert/Vectara_Upsert.ts +++ b/packages/components/nodes/vectorstores/Vectara_Upsert/Vectara_Upsert.ts @@ -1,7 +1,7 @@ import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' import { Embeddings } from 'langchain/embeddings/base' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' -import { VectaraStore, VectaraLibArgs, VectaraFilter } from 'langchain/vectorstores/vectara' +import { VectaraStore, VectaraLibArgs, VectaraFilter, VectaraContextConfig } from 'langchain/vectorstores/vectara' import { Document } from 'langchain/document' import { flatten } from 'lodash' @@ -49,6 +49,20 @@ class VectaraUpsert_VectorStores implements INode { additionalParams: true, optional: true }, + { + label: 'Sentences Before', + name: 'sentencesBefore', + type: 'number', + additionalParams: true, + optional: true + }, + { + label: 'Sentences After', + name: 'sentencesAfter', + type: 'number', + additionalParams: true, + optional: true + }, { label: 'Lambda', name: 'lambda', @@ -88,6 +102,8 @@ class VectaraUpsert_VectorStores implements INode { const docs = nodeData.inputs?.document as Document[] const embeddings = {} as Embeddings const vectaraMetadataFilter = nodeData.inputs?.filter as string + const sentencesBefore = nodeData.inputs?.sentencesBefore as number + const sentencesAfter = nodeData.inputs?.sentencesAfter as number const lambda = nodeData.inputs?.lambda as number const output = nodeData.outputs?.output as string const topK = nodeData.inputs?.topK as string @@ -103,6 +119,11 @@ class VectaraUpsert_VectorStores implements INode { if (vectaraMetadataFilter) vectaraFilter.filter = vectaraMetadataFilter if (lambda) vectaraFilter.lambda = lambda + const vectaraContextConfig: VectaraContextConfig = {} + if (sentencesBefore) vectaraContextConfig.sentencesBefore = sentencesBefore + if (sentencesAfter) vectaraContextConfig.sentencesAfter = sentencesAfter + vectaraFilter.contextConfig = vectaraContextConfig + const flattenDocs = docs && docs.length ? flatten(docs) : [] const finalDocs = [] for (let i = 0; i < flattenDocs.length; i += 1) { diff --git a/packages/components/package.json b/packages/components/package.json index bad9fb74..da4d0971 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -40,7 +40,7 @@ "google-auth-library": "^9.0.0", "graphql": "^16.6.0", "html-to-text": "^9.0.5", - "langchain": "^0.0.122", + "langchain": "^0.0.126", "linkifyjs": "^4.1.1", "mammoth": "^1.5.1", "moment": "^2.29.3", From 2e8bbaeab1fb1e8bedab6afe77272eac126a1d94 Mon Sep 17 00:00:00 2001 From: Seif Date: Tue, 15 Aug 2023 11:50:37 -0700 Subject: [PATCH 2/2] Add descriptions --- .../nodes/vectorstores/Vectara_Existing/Vectara_Existing.ts | 4 ++++ .../nodes/vectorstores/Vectara_Upsert/Vectara_Upsert.ts | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/packages/components/nodes/vectorstores/Vectara_Existing/Vectara_Existing.ts b/packages/components/nodes/vectorstores/Vectara_Existing/Vectara_Existing.ts index 80fd0639..3ef04f07 100644 --- a/packages/components/nodes/vectorstores/Vectara_Existing/Vectara_Existing.ts +++ b/packages/components/nodes/vectorstores/Vectara_Existing/Vectara_Existing.ts @@ -43,6 +43,7 @@ class VectaraExisting_VectorStores implements INode { { label: 'Sentences Before', name: 'sentencesBefore', + description: 'Number of sentences to fetch before the matched sentence. Defaults to 2.', type: 'number', additionalParams: true, optional: true @@ -50,6 +51,7 @@ class VectaraExisting_VectorStores implements INode { { label: 'Sentences After', name: 'sentencesAfter', + description: 'Number of sentences to fetch after the matched sentence. Defaults to 2.', type: 'number', additionalParams: true, optional: true @@ -57,6 +59,8 @@ class VectaraExisting_VectorStores implements INode { { label: 'Lambda', name: 'lambda', + description: + 'Improves retrieval accuracy by adjusting the balance (from 0 to 1) between neural search and keyword-based search factors.', type: 'number', additionalParams: true, optional: true diff --git a/packages/components/nodes/vectorstores/Vectara_Upsert/Vectara_Upsert.ts b/packages/components/nodes/vectorstores/Vectara_Upsert/Vectara_Upsert.ts index cda03240..51fb67ed 100644 --- a/packages/components/nodes/vectorstores/Vectara_Upsert/Vectara_Upsert.ts +++ b/packages/components/nodes/vectorstores/Vectara_Upsert/Vectara_Upsert.ts @@ -52,6 +52,7 @@ class VectaraUpsert_VectorStores implements INode { { label: 'Sentences Before', name: 'sentencesBefore', + description: 'Number of sentences to fetch before the matched sentence. Defaults to 2.', type: 'number', additionalParams: true, optional: true @@ -59,6 +60,7 @@ class VectaraUpsert_VectorStores implements INode { { label: 'Sentences After', name: 'sentencesAfter', + description: 'Number of sentences to fetch after the matched sentence. Defaults to 2.', type: 'number', additionalParams: true, optional: true @@ -66,6 +68,8 @@ class VectaraUpsert_VectorStores implements INode { { label: 'Lambda', name: 'lambda', + description: + 'Improves retrieval accuracy by adjusting the balance (from 0 to 1) between neural search and keyword-based search factors.', type: 'number', additionalParams: true, optional: true