Merge branch 'FlowiseAI:main' into Feature/Postgres-Vector-Store

2026-06-28 15:00:57 +03:00 · 2023-08-18 18:41:25 +01:00
parent be147a6e4d 5512b7d013
commit a6bdda5f2c
34 changed files with 2485 additions and 281 deletions
@@ -2,6 +2,7 @@ import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/I
 import { FaissStore } from 'langchain/vectorstores/faiss'
 import { Embeddings } from 'langchain/embeddings/base'
 import { getBaseClasses } from '../../../src/utils'
+import { Document } from 'langchain/document'

 class Faiss_Existing_VectorStores implements INode {
    label: string
@@ -70,6 +71,23 @@ class Faiss_Existing_VectorStores implements INode {

        const vectorStore = await FaissStore.load(basePath, embeddings)

+        // Avoid illegal invocation error
+        vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number) => {
+            const index = vectorStore.index
+
+            if (k > index.ntotal()) {
+                const total = index.ntotal()
+                console.warn(`k (${k}) is greater than the number of elements in the index (${total}), setting k to ${total}`)
+                k = total
+            }
+
+            const result = index.search(query, k)
+            return result.labels.map((id, index) => {
+                const uuid = vectorStore._mapping[id]
+                return [vectorStore.docstore.search(uuid), result.distances[index]] as [Document, number]
+            })
+        }
+
        if (output === 'retriever') {
            const retriever = vectorStore.asRetriever(k)
            return retriever
@@ -86,6 +86,23 @@ class FaissUpsert_VectorStores implements INode {
        const vectorStore = await FaissStore.fromDocuments(finalDocs, embeddings)
        await vectorStore.save(basePath)

+        // Avoid illegal invocation error
+        vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number) => {
+            const index = vectorStore.index
+
+            if (k > index.ntotal()) {
+                const total = index.ntotal()
+                console.warn(`k (${k}) is greater than the number of elements in the index (${total}), setting k to ${total}`)
+                k = total
+            }
+
+            const result = index.search(query, k)
+            return result.labels.map((id, index) => {
+                const uuid = vectorStore._mapping[id]
+                return [vectorStore.docstore.search(uuid), result.distances[index]] as [Document, number]
+            })
+        }
+
        if (output === 'retriever') {
            const retriever = vectorStore.asRetriever(k)
            return retriever
@@ -0,0 +1,185 @@
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { DataType, ErrorCode } from '@zilliz/milvus2-sdk-node'
+import { MilvusLibArgs, Milvus } from 'langchain/vectorstores/milvus'
+import { Embeddings } from 'langchain/embeddings/base'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { Document } from 'langchain/document'
+
+class Milvus_Existing_VectorStores implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    credential: INodeParams
+    outputs: INodeOutputsValue[]
+
+    constructor() {
+        this.label = 'Milvus Load Existing collection'
+        this.name = 'milvusExistingCollection'
+        this.version = 1.0
+        this.type = 'Milvus'
+        this.icon = 'milvus.svg'
+        this.category = 'Vector Stores'
+        this.description = 'Load existing collection from Milvus (i.e: Document has been upserted)'
+        this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            optional: true,
+            credentialNames: ['milvusAuth']
+        }
+        this.inputs = [
+            {
+                label: 'Embeddings',
+                name: 'embeddings',
+                type: 'Embeddings'
+            },
+            {
+                label: 'Milvus Server URL',
+                name: 'milvusServerUrl',
+                type: 'string',
+                placeholder: 'http://localhost:19530'
+            },
+            {
+                label: 'Milvus Collection Name',
+                name: 'milvusCollection',
+                type: 'string'
+            }
+        ]
+        this.outputs = [
+            {
+                label: 'Milvus Retriever',
+                name: 'retriever',
+                baseClasses: this.baseClasses
+            },
+            {
+                label: 'Milvus Vector Store',
+                name: 'vectorStore',
+                baseClasses: [this.type, ...getBaseClasses(Milvus)]
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        // server setup
+        const address = nodeData.inputs?.milvusServerUrl as string
+        const collectionName = nodeData.inputs?.milvusCollection as string
+
+        // embeddings
+        const embeddings = nodeData.inputs?.embeddings as Embeddings
+        const topK = nodeData.inputs?.topK as string
+
+        // output
+        const output = nodeData.outputs?.output as string
+
+        // format data
+        const k = topK ? parseInt(topK, 10) : 4
+
+        // credential
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const milvusUser = getCredentialParam('milvusUser', credentialData, nodeData)
+        const milvusPassword = getCredentialParam('milvusPassword', credentialData, nodeData)
+
+        // init MilvusLibArgs
+        const milVusArgs: MilvusLibArgs = {
+            url: address,
+            collectionName: collectionName
+        }
+
+        if (milvusUser) milVusArgs.username = milvusUser
+        if (milvusPassword) milVusArgs.password = milvusPassword
+
+        const vectorStore = await Milvus.fromExistingCollection(embeddings, milVusArgs)
+
+        // Avoid Illegal Invocation
+        vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: string) => {
+            const hasColResp = await vectorStore.client.hasCollection({
+                collection_name: vectorStore.collectionName
+            })
+            if (hasColResp.status.error_code !== ErrorCode.SUCCESS) {
+                throw new Error(`Error checking collection: ${hasColResp}`)
+            }
+            if (hasColResp.value === false) {
+                throw new Error(`Collection not found: ${vectorStore.collectionName}, please create collection before search.`)
+            }
+
+            const filterStr = filter ?? ''
+
+            await vectorStore.grabCollectionFields()
+
+            const loadResp = await vectorStore.client.loadCollectionSync({
+                collection_name: vectorStore.collectionName
+            })
+
+            if (loadResp.error_code !== ErrorCode.SUCCESS) {
+                throw new Error(`Error loading collection: ${loadResp}`)
+            }
+
+            const outputFields = vectorStore.fields.filter((field) => field !== vectorStore.vectorField)
+
+            const searchResp = await vectorStore.client.search({
+                collection_name: vectorStore.collectionName,
+                search_params: {
+                    anns_field: vectorStore.vectorField,
+                    topk: k.toString(),
+                    metric_type: vectorStore.indexCreateParams.metric_type,
+                    params: vectorStore.indexSearchParams
+                },
+                output_fields: outputFields,
+                vector_type: DataType.FloatVector,
+                vectors: [query],
+                filter: filterStr
+            })
+            if (searchResp.status.error_code !== ErrorCode.SUCCESS) {
+                throw new Error(`Error searching data: ${JSON.stringify(searchResp)}`)
+            }
+            const results: [Document, number][] = []
+            searchResp.results.forEach((result) => {
+                const fields = {
+                    pageContent: '',
+                    metadata: {} as Record<string, any>
+                }
+                Object.keys(result).forEach((key) => {
+                    if (key === vectorStore.textField) {
+                        fields.pageContent = result[key]
+                    } else if (vectorStore.fields.includes(key) || key === vectorStore.primaryField) {
+                        if (typeof result[key] === 'string') {
+                            const { isJson, obj } = checkJsonString(result[key])
+                            fields.metadata[key] = isJson ? obj : result[key]
+                        } else {
+                            fields.metadata[key] = result[key]
+                        }
+                    }
+                })
+                results.push([new Document(fields), result.score])
+            })
+            return results
+        }
+
+        if (output === 'retriever') {
+            const retriever = vectorStore.asRetriever(k)
+            return retriever
+        } else if (output === 'vectorStore') {
+            ;(vectorStore as any).k = k
+            return vectorStore
+        }
+        return vectorStore
+    }
+}
+
+function checkJsonString(value: string): { isJson: boolean; obj: any } {
+    try {
+        const result = JSON.parse(value)
+        return { isJson: true, obj: result }
+    } catch (e) {
+        return { isJson: false, obj: null }
+    }
+}
+
+module.exports = { nodeClass: Milvus_Existing_VectorStores }
@@ -0,0 +1,281 @@
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { DataType, ErrorCode, MetricType, IndexType } from '@zilliz/milvus2-sdk-node'
+import { MilvusLibArgs, Milvus } from 'langchain/vectorstores/milvus'
+import { Embeddings } from 'langchain/embeddings/base'
+import { Document } from 'langchain/document'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { flatten } from 'lodash'
+
+interface InsertRow {
+    [x: string]: string | number[]
+}
+
+class Milvus_Upsert_VectorStores implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    credential: INodeParams
+    outputs: INodeOutputsValue[]
+
+    constructor() {
+        this.label = 'Milvus Upsert Document'
+        this.name = 'milvusUpsert'
+        this.version = 1.0
+        this.type = 'Milvus'
+        this.icon = 'milvus.svg'
+        this.category = 'Vector Stores'
+        this.description = 'Upsert documents to Milvus'
+        this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            optional: true,
+            credentialNames: ['milvusAuth']
+        }
+        this.inputs = [
+            {
+                label: 'Document',
+                name: 'document',
+                type: 'Document',
+                list: true
+            },
+            {
+                label: 'Embeddings',
+                name: 'embeddings',
+                type: 'Embeddings'
+            },
+            {
+                label: 'Milvus Server URL',
+                name: 'milvusServerUrl',
+                type: 'string',
+                placeholder: 'http://localhost:19530'
+            },
+            {
+                label: 'Milvus Collection Name',
+                name: 'milvusCollection',
+                type: 'string'
+            }
+        ]
+        this.outputs = [
+            {
+                label: 'Milvus Retriever',
+                name: 'retriever',
+                baseClasses: this.baseClasses
+            },
+            {
+                label: 'Milvus Vector Store',
+                name: 'vectorStore',
+                baseClasses: [this.type, ...getBaseClasses(Milvus)]
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        // server setup
+        const address = nodeData.inputs?.milvusServerUrl as string
+        const collectionName = nodeData.inputs?.milvusCollection as string
+
+        // embeddings
+        const docs = nodeData.inputs?.document as Document[]
+        const embeddings = nodeData.inputs?.embeddings as Embeddings
+        const topK = nodeData.inputs?.topK as string
+
+        // output
+        const output = nodeData.outputs?.output as string
+
+        // format data
+        const k = topK ? parseInt(topK, 10) : 4
+
+        // credential
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const milvusUser = getCredentialParam('milvusUser', credentialData, nodeData)
+        const milvusPassword = getCredentialParam('milvusPassword', credentialData, nodeData)
+
+        // init MilvusLibArgs
+        const milVusArgs: MilvusLibArgs = {
+            url: address,
+            collectionName: collectionName
+        }
+
+        if (milvusUser) milVusArgs.username = milvusUser
+        if (milvusPassword) milVusArgs.password = milvusPassword
+
+        const flattenDocs = docs && docs.length ? flatten(docs) : []
+        const finalDocs = []
+        for (let i = 0; i < flattenDocs.length; i += 1) {
+            finalDocs.push(new Document(flattenDocs[i]))
+        }
+
+        const vectorStore = await MilvusUpsert.fromDocuments(finalDocs, embeddings, milVusArgs)
+
+        // Avoid Illegal Invocation
+        vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: string) => {
+            const hasColResp = await vectorStore.client.hasCollection({
+                collection_name: vectorStore.collectionName
+            })
+            if (hasColResp.status.error_code !== ErrorCode.SUCCESS) {
+                throw new Error(`Error checking collection: ${hasColResp}`)
+            }
+            if (hasColResp.value === false) {
+                throw new Error(`Collection not found: ${vectorStore.collectionName}, please create collection before search.`)
+            }
+
+            const filterStr = filter ?? ''
+
+            await vectorStore.grabCollectionFields()
+
+            const loadResp = await vectorStore.client.loadCollectionSync({
+                collection_name: vectorStore.collectionName
+            })
+            if (loadResp.error_code !== ErrorCode.SUCCESS) {
+                throw new Error(`Error loading collection: ${loadResp}`)
+            }
+
+            const outputFields = vectorStore.fields.filter((field) => field !== vectorStore.vectorField)
+
+            const searchResp = await vectorStore.client.search({
+                collection_name: vectorStore.collectionName,
+                search_params: {
+                    anns_field: vectorStore.vectorField,
+                    topk: k.toString(),
+                    metric_type: vectorStore.indexCreateParams.metric_type,
+                    params: vectorStore.indexSearchParams
+                },
+                output_fields: outputFields,
+                vector_type: DataType.FloatVector,
+                vectors: [query],
+                filter: filterStr
+            })
+            if (searchResp.status.error_code !== ErrorCode.SUCCESS) {
+                throw new Error(`Error searching data: ${JSON.stringify(searchResp)}`)
+            }
+            const results: [Document, number][] = []
+            searchResp.results.forEach((result) => {
+                const fields = {
+                    pageContent: '',
+                    metadata: {} as Record<string, any>
+                }
+                Object.keys(result).forEach((key) => {
+                    if (key === vectorStore.textField) {
+                        fields.pageContent = result[key]
+                    } else if (vectorStore.fields.includes(key) || key === vectorStore.primaryField) {
+                        if (typeof result[key] === 'string') {
+                            const { isJson, obj } = checkJsonString(result[key])
+                            fields.metadata[key] = isJson ? obj : result[key]
+                        } else {
+                            fields.metadata[key] = result[key]
+                        }
+                    }
+                })
+                results.push([new Document(fields), result.score])
+            })
+            return results
+        }
+
+        if (output === 'retriever') {
+            const retriever = vectorStore.asRetriever(k)
+            return retriever
+        } else if (output === 'vectorStore') {
+            ;(vectorStore as any).k = k
+            return vectorStore
+        }
+        return vectorStore
+    }
+}
+
+function checkJsonString(value: string): { isJson: boolean; obj: any } {
+    try {
+        const result = JSON.parse(value)
+        return { isJson: true, obj: result }
+    } catch (e) {
+        return { isJson: false, obj: null }
+    }
+}
+
+class MilvusUpsert extends Milvus {
+    async addVectors(vectors: number[][], documents: Document[]): Promise<void> {
+        if (vectors.length === 0) {
+            return
+        }
+        await this.ensureCollection(vectors, documents)
+
+        const insertDatas: InsertRow[] = []
+
+        for (let index = 0; index < vectors.length; index++) {
+            const vec = vectors[index]
+            const doc = documents[index]
+            const data: InsertRow = {
+                [this.textField]: doc.pageContent,
+                [this.vectorField]: vec
+            }
+            this.fields.forEach((field) => {
+                switch (field) {
+                    case this.primaryField:
+                        if (!this.autoId) {
+                            if (doc.metadata[this.primaryField] === undefined) {
+                                throw new Error(
+                                    `The Collection's primaryField is configured with autoId=false, thus its value must be provided through metadata.`
+                                )
+                            }
+                            data[field] = doc.metadata[this.primaryField]
+                        }
+                        break
+                    case this.textField:
+                        data[field] = doc.pageContent
+                        break
+                    case this.vectorField:
+                        data[field] = vec
+                        break
+                    default: // metadata fields
+                        if (doc.metadata[field] === undefined) {
+                            throw new Error(`The field "${field}" is not provided in documents[${index}].metadata.`)
+                        } else if (typeof doc.metadata[field] === 'object') {
+                            data[field] = JSON.stringify(doc.metadata[field])
+                        } else {
+                            data[field] = doc.metadata[field]
+                        }
+                        break
+                }
+            })
+
+            insertDatas.push(data)
+        }
+
+        const descIndexResp = await this.client.describeIndex({
+            collection_name: this.collectionName
+        })
+
+        if (descIndexResp.status.error_code === ErrorCode.INDEX_NOT_EXIST) {
+            const resp = await this.client.createIndex({
+                collection_name: this.collectionName,
+                field_name: this.vectorField,
+                index_name: `myindex_${Date.now().toString()}`,
+                index_type: IndexType.AUTOINDEX,
+                metric_type: MetricType.L2
+            })
+            if (resp.error_code !== ErrorCode.SUCCESS) {
+                throw new Error(`Error creating index`)
+            }
+        }
+
+        const insertResp = await this.client.insert({
+            collection_name: this.collectionName,
+            fields_data: insertDatas
+        })
+
+        if (insertResp.status.error_code !== ErrorCode.SUCCESS) {
+            throw new Error(`Error inserting data: ${JSON.stringify(insertResp)}`)
+        }
+
+        await this.client.flushSync({ collection_names: [this.collectionName] })
+    }
+}
+
+module.exports = { nodeClass: Milvus_Upsert_VectorStores }
@@ -0,0 +1,5 @@
+<svg width="362" height="246" viewBox="0 0 362 246" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M255.407 36.2761C207.644 -11.661 130.191 -11.661 82.427 36.2761L4.35891 114.626C-0.255949 119.262 -0.255949 126.728 4.35891 131.364L82.427 209.714C130.191 257.651 207.644 257.651 255.407 209.793C303.248 161.934 303.248 84.2132 255.407 36.2761ZM236.948 186.767C201.875 221.973 144.958 221.973 109.885 186.767L52.4304 129.164C49.0461 125.785 49.0461 120.284 52.4304 116.826L109.808 59.3016C144.881 24.0953 201.798 24.0953 236.871 59.3016C272.021 94.5078 272.021 151.561 236.948 186.767Z" fill="#00B3FF"/>
+<path d="M357.699 114.704L323.318 79.5765C321.241 77.4547 317.78 79.4193 318.472 82.327C324.395 109.125 324.395 137.101 318.472 163.899C317.857 166.806 321.318 168.692 323.318 166.649L357.699 131.521C362.237 126.806 362.237 119.341 357.699 114.704Z" fill="#00B3FF"/>
+<path d="M173.799 184.646C207.059 184.646 234.023 157.097 234.023 123.113C234.023 89.13 207.059 61.5811 173.799 61.5811C140.538 61.5811 113.575 89.13 113.575 123.113C113.575 157.097 140.538 184.646 173.799 184.646Z" fill="#00B3FF"/>
+</svg>
@@ -1,6 +1,6 @@
 import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
-import { VectaraStore, VectaraLibArgs, VectaraFilter } from 'langchain/vectorstores/vectara'
+import { VectaraStore, VectaraLibArgs, VectaraFilter, VectaraContextConfig } from 'langchain/vectorstores/vectara'

 class VectaraExisting_VectorStores implements INode {
    label: string
@@ -40,9 +40,27 @@ class VectaraExisting_VectorStores implements INode {
                additionalParams: true,
                optional: true
            },
+            {
+                label: 'Sentences Before',
+                name: 'sentencesBefore',
+                description: 'Number of sentences to fetch before the matched sentence. Defaults to 2.',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Sentences After',
+                name: 'sentencesAfter',
+                description: 'Number of sentences to fetch after the matched sentence. Defaults to 2.',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            },
            {
                label: 'Lambda',
                name: 'lambda',
+                description:
+                    'Improves retrieval accuracy by adjusting the balance (from 0 to 1) between neural search and keyword-based search factors.',
                type: 'number',
                additionalParams: true,
                optional: true
@@ -77,6 +95,8 @@ class VectaraExisting_VectorStores implements INode {
        const corpusId = getCredentialParam('corpusID', credentialData, nodeData)

        const vectaraMetadataFilter = nodeData.inputs?.filter as string
+        const sentencesBefore = nodeData.inputs?.sentencesBefore as number
+        const sentencesAfter = nodeData.inputs?.sentencesAfter as number
        const lambda = nodeData.inputs?.lambda as number
        const output = nodeData.outputs?.output as string
        const topK = nodeData.inputs?.topK as string
@@ -92,6 +112,11 @@ class VectaraExisting_VectorStores implements INode {
        if (vectaraMetadataFilter) vectaraFilter.filter = vectaraMetadataFilter
        if (lambda) vectaraFilter.lambda = lambda

+        const vectaraContextConfig: VectaraContextConfig = {}
+        if (sentencesBefore) vectaraContextConfig.sentencesBefore = sentencesBefore
+        if (sentencesAfter) vectaraContextConfig.sentencesAfter = sentencesAfter
+        vectaraFilter.contextConfig = vectaraContextConfig
+
        const vectorStore = new VectaraStore(vectaraArgs)

        if (output === 'retriever') {
@@ -1,7 +1,7 @@
 import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
 import { Embeddings } from 'langchain/embeddings/base'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
-import { VectaraStore, VectaraLibArgs, VectaraFilter } from 'langchain/vectorstores/vectara'
+import { VectaraStore, VectaraLibArgs, VectaraFilter, VectaraContextConfig } from 'langchain/vectorstores/vectara'
 import { Document } from 'langchain/document'
 import { flatten } from 'lodash'

@@ -49,9 +49,27 @@ class VectaraUpsert_VectorStores implements INode {
                additionalParams: true,
                optional: true
            },
+            {
+                label: 'Sentences Before',
+                name: 'sentencesBefore',
+                description: 'Number of sentences to fetch before the matched sentence. Defaults to 2.',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Sentences After',
+                name: 'sentencesAfter',
+                description: 'Number of sentences to fetch after the matched sentence. Defaults to 2.',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            },
            {
                label: 'Lambda',
                name: 'lambda',
+                description:
+                    'Improves retrieval accuracy by adjusting the balance (from 0 to 1) between neural search and keyword-based search factors.',
                type: 'number',
                additionalParams: true,
                optional: true
@@ -88,6 +106,8 @@ class VectaraUpsert_VectorStores implements INode {
        const docs = nodeData.inputs?.document as Document[]
        const embeddings = {} as Embeddings
        const vectaraMetadataFilter = nodeData.inputs?.filter as string
+        const sentencesBefore = nodeData.inputs?.sentencesBefore as number
+        const sentencesAfter = nodeData.inputs?.sentencesAfter as number
        const lambda = nodeData.inputs?.lambda as number
        const output = nodeData.outputs?.output as string
        const topK = nodeData.inputs?.topK as string
@@ -103,6 +123,11 @@ class VectaraUpsert_VectorStores implements INode {
        if (vectaraMetadataFilter) vectaraFilter.filter = vectaraMetadataFilter
        if (lambda) vectaraFilter.lambda = lambda

+        const vectaraContextConfig: VectaraContextConfig = {}
+        if (sentencesBefore) vectaraContextConfig.sentencesBefore = sentencesBefore
+        if (sentencesAfter) vectaraContextConfig.sentencesAfter = sentencesAfter
+        vectaraFilter.contextConfig = vectaraContextConfig
+
        const flattenDocs = docs && docs.length ? flatten(docs) : []
        const finalDocs = []
        for (let i = 0; i < flattenDocs.length; i += 1) {
@@ -0,0 +1,235 @@
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ZepVectorStore, IZepConfig } from 'langchain/vectorstores/zep'
+import { Embeddings } from 'langchain/embeddings/base'
+import { Document } from 'langchain/document'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { IDocument, ZepClient } from '@getzep/zep-js'
+
+class Zep_Existing_VectorStores implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    credential: INodeParams
+    outputs: INodeOutputsValue[]
+
+    constructor() {
+        this.label = 'Zep Load Existing Index'
+        this.name = 'zepExistingIndex'
+        this.version = 1.0
+        this.type = 'Zep'
+        this.icon = 'zep.png'
+        this.category = 'Vector Stores'
+        this.description = 'Load existing index from Zep (i.e: Document has been upserted)'
+        this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            optional: true,
+            description: 'Configure JWT authentication on your Zep instance (Optional)',
+            credentialNames: ['zepMemoryApi']
+        }
+        this.inputs = [
+            {
+                label: 'Embeddings',
+                name: 'embeddings',
+                type: 'Embeddings'
+            },
+            {
+                label: 'Base URL',
+                name: 'baseURL',
+                type: 'string',
+                default: 'http://127.0.0.1:8000'
+            },
+            {
+                label: 'Zep Collection',
+                name: 'zepCollection',
+                type: 'string',
+                placeholder: 'my-first-collection'
+            },
+            {
+                label: 'Zep Metadata Filter',
+                name: 'zepMetadataFilter',
+                type: 'json',
+                optional: true,
+                additionalParams: true
+            },
+            {
+                label: 'Embedding Dimension',
+                name: 'dimension',
+                type: 'number',
+                default: 1536,
+                additionalParams: true
+            },
+            {
+                label: 'Top K',
+                name: 'topK',
+                description: 'Number of top results to fetch. Default to 4',
+                placeholder: '4',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            }
+        ]
+        this.outputs = [
+            {
+                label: 'Pinecone Retriever',
+                name: 'retriever',
+                baseClasses: this.baseClasses
+            },
+            {
+                label: 'Pinecone Vector Store',
+                name: 'vectorStore',
+                baseClasses: [this.type, ...getBaseClasses(ZepVectorStore)]
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const baseURL = nodeData.inputs?.baseURL as string
+        const zepCollection = nodeData.inputs?.zepCollection as string
+        const zepMetadataFilter = nodeData.inputs?.zepMetadataFilter
+        const dimension = nodeData.inputs?.dimension as number
+        const embeddings = nodeData.inputs?.embeddings as Embeddings
+        const output = nodeData.outputs?.output as string
+        const topK = nodeData.inputs?.topK as string
+        const k = topK ? parseFloat(topK) : 4
+
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
+
+        const zepConfig: IZepConfig & Partial<ZepFilter> = {
+            apiUrl: baseURL,
+            collectionName: zepCollection,
+            embeddingDimensions: dimension,
+            isAutoEmbedded: false
+        }
+        if (apiKey) zepConfig.apiKey = apiKey
+        if (zepMetadataFilter) {
+            const metadatafilter = typeof zepMetadataFilter === 'object' ? zepMetadataFilter : JSON.parse(zepMetadataFilter)
+            zepConfig.filter = metadatafilter
+        }
+
+        const vectorStore = await ZepExistingVS.fromExistingIndex(embeddings, zepConfig)
+
+        if (output === 'retriever') {
+            const retriever = vectorStore.asRetriever(k)
+            return retriever
+        } else if (output === 'vectorStore') {
+            ;(vectorStore as any).k = k
+            return vectorStore
+        }
+        return vectorStore
+    }
+}
+
+interface ZepFilter {
+    filter: Record<string, any>
+}
+
+function zepDocsToDocumentsAndScore(results: IDocument[]): [Document, number][] {
+    return results.map((d) => [
+        new Document({
+            pageContent: d.content,
+            metadata: d.metadata
+        }),
+        d.score ? d.score : 0
+    ])
+}
+
+function assignMetadata(value: string | Record<string, unknown> | object | undefined): Record<string, unknown> | undefined {
+    if (typeof value === 'object' && value !== null) {
+        return value as Record<string, unknown>
+    }
+    if (value !== undefined) {
+        console.warn('Metadata filters must be an object, Record, or undefined.')
+    }
+    return undefined
+}
+
+class ZepExistingVS extends ZepVectorStore {
+    filter?: Record<string, any>
+    args?: IZepConfig & Partial<ZepFilter>
+
+    constructor(embeddings: Embeddings, args: IZepConfig & Partial<ZepFilter>) {
+        super(embeddings, args)
+        this.filter = args.filter
+        this.args = args
+    }
+
+    async initalizeCollection(args: IZepConfig & Partial<ZepFilter>) {
+        this.client = await ZepClient.init(args.apiUrl, args.apiKey)
+        try {
+            this.collection = await this.client.document.getCollection(args.collectionName)
+        } catch (err) {
+            if (err instanceof Error) {
+                if (err.name === 'NotFoundError') {
+                    await this.createNewCollection(args)
+                } else {
+                    throw err
+                }
+            }
+        }
+    }
+
+    async createNewCollection(args: IZepConfig & Partial<ZepFilter>) {
+        if (!args.embeddingDimensions) {
+            throw new Error(
+                `Collection ${args.collectionName} not found. You can create a new Collection by providing embeddingDimensions.`
+            )
+        }
+
+        this.collection = await this.client.document.addCollection({
+            name: args.collectionName,
+            description: args.description,
+            metadata: args.metadata,
+            embeddingDimensions: args.embeddingDimensions,
+            isAutoEmbedded: false
+        })
+    }
+
+    async similaritySearchVectorWithScore(
+        query: number[],
+        k: number,
+        filter?: Record<string, unknown> | undefined
+    ): Promise<[Document, number][]> {
+        if (filter && this.filter) {
+            throw new Error('cannot provide both `filter` and `this.filter`')
+        }
+        const _filters = filter ?? this.filter
+        const ANDFilters = []
+        for (const filterKey in _filters) {
+            let filterVal = _filters[filterKey]
+            if (typeof filterVal === 'string') filterVal = `"${filterVal}"`
+            ANDFilters.push({ jsonpath: `$[*] ? (@.${filterKey} == ${filterVal})` })
+        }
+        const newfilter = {
+            where: { and: ANDFilters }
+        }
+        await this.initalizeCollection(this.args!).catch((err) => {
+            console.error('Error initializing collection:', err)
+            throw err
+        })
+        const results = await this.collection.search(
+            {
+                embedding: new Float32Array(query),
+                metadata: assignMetadata(newfilter)
+            },
+            k
+        )
+        return zepDocsToDocumentsAndScore(results)
+    }
+
+    static async fromExistingIndex(embeddings: Embeddings, dbConfig: IZepConfig & Partial<ZepFilter>): Promise<ZepVectorStore> {
+        const instance = new this(embeddings, dbConfig)
+        return instance
+    }
+}
+
+module.exports = { nodeClass: Zep_Existing_VectorStores }
@@ -0,0 +1,133 @@
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ZepVectorStore, IZepConfig } from 'langchain/vectorstores/zep'
+import { Embeddings } from 'langchain/embeddings/base'
+import { Document } from 'langchain/document'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { flatten } from 'lodash'
+
+class Zep_Upsert_VectorStores implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    credential: INodeParams
+    outputs: INodeOutputsValue[]
+
+    constructor() {
+        this.label = 'Zep Upsert Document'
+        this.name = 'zepUpsert'
+        this.version = 1.0
+        this.type = 'Zep'
+        this.icon = 'zep.png'
+        this.category = 'Vector Stores'
+        this.description = 'Upsert documents to Zep'
+        this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            optional: true,
+            description: 'Configure JWT authentication on your Zep instance (Optional)',
+            credentialNames: ['zepMemoryApi']
+        }
+        this.inputs = [
+            {
+                label: 'Document',
+                name: 'document',
+                type: 'Document',
+                list: true
+            },
+            {
+                label: 'Embeddings',
+                name: 'embeddings',
+                type: 'Embeddings'
+            },
+            {
+                label: 'Base URL',
+                name: 'baseURL',
+                type: 'string',
+                default: 'http://127.0.0.1:8000'
+            },
+            {
+                label: 'Zep Collection',
+                name: 'zepCollection',
+                type: 'string',
+                placeholder: 'my-first-collection'
+            },
+            {
+                label: 'Embedding Dimension',
+                name: 'dimension',
+                type: 'number',
+                default: 1536,
+                additionalParams: true
+            },
+            {
+                label: 'Top K',
+                name: 'topK',
+                description: 'Number of top results to fetch. Default to 4',
+                placeholder: '4',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            }
+        ]
+        this.outputs = [
+            {
+                label: 'Zep Retriever',
+                name: 'retriever',
+                baseClasses: this.baseClasses
+            },
+            {
+                label: 'Zep Vector Store',
+                name: 'vectorStore',
+                baseClasses: [this.type, ...getBaseClasses(ZepVectorStore)]
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const baseURL = nodeData.inputs?.baseURL as string
+        const zepCollection = nodeData.inputs?.zepCollection as string
+        const dimension = (nodeData.inputs?.dimension as number) ?? 1536
+        const docs = nodeData.inputs?.document as Document[]
+        const embeddings = nodeData.inputs?.embeddings as Embeddings
+        const topK = nodeData.inputs?.topK as string
+        const k = topK ? parseFloat(topK) : 4
+        const output = nodeData.outputs?.output as string
+
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
+
+        const flattenDocs = docs && docs.length ? flatten(docs) : []
+        const finalDocs = []
+        for (let i = 0; i < flattenDocs.length; i += 1) {
+            finalDocs.push(new Document(flattenDocs[i]))
+        }
+
+        const zepConfig: IZepConfig = {
+            apiUrl: baseURL,
+            collectionName: zepCollection,
+            embeddingDimensions: dimension,
+            isAutoEmbedded: false
+        }
+        if (apiKey) zepConfig.apiKey = apiKey
+
+        const vectorStore = await ZepVectorStore.fromDocuments(finalDocs, embeddings, zepConfig)
+
+        if (output === 'retriever') {
+            const retriever = vectorStore.asRetriever(k)
+            return retriever
+        } else if (output === 'vectorStore') {
+            ;(vectorStore as any).k = k
+            return vectorStore
+        }
+        return vectorStore
+    }
+}
+
+module.exports = { nodeClass: Zep_Upsert_VectorStores }