From 0e6e5476cc2695a06467257b75436cda314b6fbf Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Wed, 31 May 2023 14:29:59 +0100 Subject: [PATCH] Revert "Feature/Add faiss" --- .../Faiss_Existing/Faiss_Existing.ts | 70 --------------- .../vectorstores/Faiss_Existing/faiss.svg | 10 --- .../vectorstores/Faiss_Upsert/Faiss_Upsert.ts | 85 ------------------- .../nodes/vectorstores/Faiss_Upsert/faiss.svg | 10 --- packages/components/package.json | 1 - packages/server/src/index.ts | 5 +- packages/server/src/utils/index.ts | 30 +------ 7 files changed, 4 insertions(+), 207 deletions(-) delete mode 100644 packages/components/nodes/vectorstores/Faiss_Existing/Faiss_Existing.ts delete mode 100644 packages/components/nodes/vectorstores/Faiss_Existing/faiss.svg delete mode 100644 packages/components/nodes/vectorstores/Faiss_Upsert/Faiss_Upsert.ts delete mode 100644 packages/components/nodes/vectorstores/Faiss_Upsert/faiss.svg diff --git a/packages/components/nodes/vectorstores/Faiss_Existing/Faiss_Existing.ts b/packages/components/nodes/vectorstores/Faiss_Existing/Faiss_Existing.ts deleted file mode 100644 index 8916a734..00000000 --- a/packages/components/nodes/vectorstores/Faiss_Existing/Faiss_Existing.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' -import { FaissStore } from 'langchain/vectorstores/faiss' -import { Embeddings } from 'langchain/embeddings/base' -import { getBaseClasses } from '../../../src/utils' - -class Faiss_Existing_VectorStores implements INode { - label: string - name: string - description: string - type: string - icon: string - category: string - baseClasses: string[] - inputs: INodeParams[] - outputs: INodeOutputsValue[] - - constructor() { - this.label = 'Faiss Load Existing Index' - this.name = 'faissExistingIndex' - this.type = 'Faiss' - this.icon = 'faiss.svg' - this.category = 'Vector Stores' - this.description = 'Load existing index from Faiss (i.e: Document has been upserted)' - this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever'] - this.inputs = [ - { - label: 'Embeddings', - name: 'embeddings', - type: 'Embeddings' - }, - { - label: 'Base Path to load', - name: 'basePath', - description: 'Path to load faiss.index file', - placeholder: `C:\\Users\\User\\Desktop`, - type: 'string' - } - ] - this.outputs = [ - { - label: 'Faiss Retriever', - name: 'retriever', - baseClasses: this.baseClasses - }, - { - label: 'Faiss Vector Store', - name: 'vectorStore', - baseClasses: [this.type, ...getBaseClasses(FaissStore)] - } - ] - } - - async init(nodeData: INodeData): Promise { - const embeddings = nodeData.inputs?.embeddings as Embeddings - const basePath = nodeData.inputs?.basePath as string - const output = nodeData.outputs?.output as string - - const vectorStore = await FaissStore.load(basePath, embeddings) - - if (output === 'retriever') { - const retriever = vectorStore.asRetriever() - return retriever - } else if (output === 'vectorStore') { - return vectorStore - } - return vectorStore - } -} - -module.exports = { nodeClass: Faiss_Existing_VectorStores } diff --git a/packages/components/nodes/vectorstores/Faiss_Existing/faiss.svg b/packages/components/nodes/vectorstores/Faiss_Existing/faiss.svg deleted file mode 100644 index 5fbe9832..00000000 --- a/packages/components/nodes/vectorstores/Faiss_Existing/faiss.svg +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - \ No newline at end of file diff --git a/packages/components/nodes/vectorstores/Faiss_Upsert/Faiss_Upsert.ts b/packages/components/nodes/vectorstores/Faiss_Upsert/Faiss_Upsert.ts deleted file mode 100644 index 2db6a038..00000000 --- a/packages/components/nodes/vectorstores/Faiss_Upsert/Faiss_Upsert.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' -import { Embeddings } from 'langchain/embeddings/base' -import { Document } from 'langchain/document' -import { getBaseClasses } from '../../../src/utils' -import { FaissStore } from 'langchain/vectorstores/faiss' - -class FaissUpsert_VectorStores implements INode { - label: string - name: string - description: string - type: string - icon: string - category: string - baseClasses: string[] - inputs: INodeParams[] - outputs: INodeOutputsValue[] - - constructor() { - this.label = 'Faiss Upsert Document' - this.name = 'faissUpsert' - this.type = 'Faiss' - this.icon = 'faiss.svg' - this.category = 'Vector Stores' - this.description = 'Upsert documents to Faiss' - this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever'] - this.inputs = [ - { - label: 'Document', - name: 'document', - type: 'Document', - list: true - }, - { - label: 'Embeddings', - name: 'embeddings', - type: 'Embeddings' - }, - { - label: 'Base Path to store', - name: 'basePath', - description: 'Path to store faiss.index file', - placeholder: `C:\\Users\\User\\Desktop`, - type: 'string' - } - ] - this.outputs = [ - { - label: 'Faiss Retriever', - name: 'retriever', - baseClasses: this.baseClasses - }, - { - label: 'Faiss Vector Store', - name: 'vectorStore', - baseClasses: [this.type, ...getBaseClasses(FaissStore)] - } - ] - } - - async init(nodeData: INodeData): Promise { - const docs = nodeData.inputs?.document as Document[] - const embeddings = nodeData.inputs?.embeddings as Embeddings - const output = nodeData.outputs?.output as string - const basePath = nodeData.inputs?.basePath as string - - const flattenDocs = docs && docs.length ? docs.flat() : [] - const finalDocs = [] - for (let i = 0; i < flattenDocs.length; i += 1) { - finalDocs.push(new Document(flattenDocs[i])) - } - - const vectorStore = await FaissStore.fromDocuments(finalDocs, embeddings) - await vectorStore.save(basePath) - - if (output === 'retriever') { - const retriever = vectorStore.asRetriever() - return retriever - } else if (output === 'vectorStore') { - return vectorStore - } - return vectorStore - } -} - -module.exports = { nodeClass: FaissUpsert_VectorStores } diff --git a/packages/components/nodes/vectorstores/Faiss_Upsert/faiss.svg b/packages/components/nodes/vectorstores/Faiss_Upsert/faiss.svg deleted file mode 100644 index 5fbe9832..00000000 --- a/packages/components/nodes/vectorstores/Faiss_Upsert/faiss.svg +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - \ No newline at end of file diff --git a/packages/components/package.json b/packages/components/package.json index 65eedd51..8f8e68bc 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -28,7 +28,6 @@ "d3-dsv": "2", "dotenv": "^16.0.0", "express": "^4.17.3", - "faiss-node": "^0.2.0", "form-data": "^4.0.0", "graphql": "^16.6.0", "langchain": "^0.0.82", diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 92cd12d0..548c27e9 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -34,8 +34,7 @@ import { findAvailableConfigs, isSameOverrideConfig, replaceAllAPIKeys, - isFlowValidForStream, - isVectorStoreFaiss + isFlowValidForStream } from './utils' import { cloneDeep } from 'lodash' import { getDataSource } from './DataSource' @@ -635,8 +634,6 @@ export class App { const nodeModule = await import(nodeInstanceFilePath) const nodeInstance = new nodeModule.nodeClass() - isStreamValid = isStreamValid && !isVectorStoreFaiss(nodeToExecuteData) - const result = isStreamValid ? await nodeInstance.run(nodeToExecuteData, incomingInput.question, { chatHistory: incomingInput.history, diff --git a/packages/server/src/utils/index.ts b/packages/server/src/utils/index.ts index 18473c51..982a82d0 100644 --- a/packages/server/src/utils/index.ts +++ b/packages/server/src/utils/index.ts @@ -14,7 +14,7 @@ import { INodeData, IOverrideConfig } from '../Interface' -import { cloneDeep, get, omit, merge } from 'lodash' +import { cloneDeep, get } from 'lodash' import { ICommonObject, getInputVariables } from 'flowise-components' import { scryptSync, randomBytes, timingSafeEqual } from 'crypto' @@ -317,25 +317,6 @@ export const getVariableValue = (paramValue: string, reactFlowNodes: IReactFlowN return returnVal } -/** - * Temporarily disable streaming if vectorStore is Faiss - * @param {INodeData} flowNodeData - * @returns {boolean} - */ -export const isVectorStoreFaiss = (flowNodeData: INodeData) => { - if (flowNodeData.inputs && flowNodeData.inputs.vectorStoreRetriever) { - const vectorStoreRetriever = flowNodeData.inputs.vectorStoreRetriever - if (typeof vectorStoreRetriever === 'string' && vectorStoreRetriever.includes('faiss')) return true - if ( - typeof vectorStoreRetriever === 'object' && - vectorStoreRetriever.vectorStore && - vectorStoreRetriever.vectorStore.constructor.name === 'FaissStore' - ) - return true - } - return false -} - /** * Loop through each inputs and resolve variable if neccessary * @param {INodeData} reactFlowNodeData @@ -344,12 +325,7 @@ export const isVectorStoreFaiss = (flowNodeData: INodeData) => { * @returns {INodeData} */ export const resolveVariables = (reactFlowNodeData: INodeData, reactFlowNodes: IReactFlowNode[], question: string): INodeData => { - let flowNodeData = cloneDeep(reactFlowNodeData) - if (reactFlowNodeData.instance && isVectorStoreFaiss(reactFlowNodeData)) { - // omit and merge because cloneDeep of instance gives "Illegal invocation" Exception - const flowNodeDataWithoutInstance = cloneDeep(omit(reactFlowNodeData, ['instance'])) - flowNodeData = merge(flowNodeDataWithoutInstance, { instance: reactFlowNodeData.instance }) - } + const flowNodeData = cloneDeep(reactFlowNodeData) const types = 'inputs' const getParamValues = (paramsObj: ICommonObject) => { @@ -657,5 +633,5 @@ export const isFlowValidForStream = (reactFlowNodes: IReactFlowNode[], endingNod } } - return isChatOrLLMsExist && endingNodeData.category === 'Chains' && !isVectorStoreFaiss(endingNodeData) + return isChatOrLLMsExist && endingNodeData.category === 'Chains' }