Document Store - Phase 2 (#2912)

* Document Store - Phase 2

* Adding additional columns for vector store config, document store phase 2

* Adding additional columns for vector store config, document store phase 2

* Document Store - Phase 2 - Upsert and Query

* ux cleanup

* retrieval settings and more ux changes

* adding MMR params to execution

* Making the upsert process async.

* add upsert history changes

* making the searchParams dynamic

* removing unnecessary params

* add ability to delete data from vector store

* update margin for vector store query

* adding option to save config in the retrieval playground

* adding chunk number for query return chunks

* Adding a Document Store node in the VectorStore category

* update doc store status, ui touchup

---------

Co-authored-by: Henry <hzj94@hotmail.com>

packages/components/nodes/vectorstores/Chroma/Chroma.ts

@@ -150,6 +150,42 @@ class Chroma_VectorStores implements INode {
             } catch (e) {
                 throw new Error(e)
             }
+        },
+        async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise<void> {
+            const collectionName = nodeData.inputs?.collectionName as string
+            const embeddings = nodeData.inputs?.embeddings as Embeddings
+            const chromaURL = nodeData.inputs?.chromaURL as string
+            const recordManager = nodeData.inputs?.recordManager
+
+            const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+            const chromaApiKey = getCredentialParam('chromaApiKey', credentialData, nodeData)
+
+            const obj: {
+                collectionName: string
+                url?: string
+                chromaApiKey?: string
+            } = { collectionName }
+            if (chromaURL) obj.url = chromaURL
+            if (chromaApiKey) obj.chromaApiKey = chromaApiKey
+
+            try {
+                if (recordManager) {
+                    const vectorStoreName = collectionName
+                    await recordManager.createSchema()
+                    ;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
+                    const keys: string[] = await recordManager.listKeys({})
+
+                    const chromaStore = new ChromaExtended(embeddings, obj)
+
+                    await chromaStore.delete({ ids: keys })
+                    await recordManager.deleteKeys(keys)
+                } else {
+                    const chromaStore = new ChromaExtended(embeddings, obj)
+                    await chromaStore.delete({ ids })
+                }
+            } catch (e) {
+                throw new Error(e)
+            }
         }
     }
 

packages/components/nodes/vectorstores/DocumentStoreVS/DocStoreVector.ts

@@ -0,0 +1,174 @@
+import { ICommonObject, IDatabaseEntity, INode, INodeData, INodeOptionsValue, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { DataSource } from 'typeorm'
+
+class DocStore_VectorStores implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    outputs: INodeOutputsValue[]
+    badge: string
+
+    constructor() {
+        this.label = 'Document Store (Vector)'
+        this.name = 'documentStoreVS'
+        this.version = 1.0
+        this.type = 'DocumentStoreVS'
+        this.icon = 'dstore.svg'
+        this.badge = 'New'
+        this.category = 'Vector Stores'
+        this.description = `Search and retrieve documents from Document Store`
+        this.baseClasses = [this.type]
+        this.inputs = [
+            {
+                label: 'Select Store',
+                name: 'selectedStore',
+                type: 'asyncOptions',
+                loadMethod: 'listStores'
+            }
+        ]
+        this.outputs = [
+            {
+                label: 'Retriever',
+                name: 'retriever',
+                baseClasses: ['BaseRetriever']
+            },
+            {
+                label: 'Vector Store',
+                name: 'vectorStore',
+                baseClasses: ['VectorStore']
+            }
+        ]
+    }
+
+    //@ts-ignore
+    loadMethods = {
+        async listStores(_: INodeData, options: ICommonObject): Promise<INodeOptionsValue[]> {
+            const returnData: INodeOptionsValue[] = []
+
+            const appDataSource = options.appDataSource as DataSource
+            const databaseEntities = options.databaseEntities as IDatabaseEntity
+
+            if (appDataSource === undefined || !appDataSource) {
+                return returnData
+            }
+
+            const stores = await appDataSource.getRepository(databaseEntities['DocumentStore']).find()
+            for (const store of stores) {
+                if (store.status === 'UPSERTED') {
+                    const obj = {
+                        name: store.id,
+                        label: store.name,
+                        description: store.description
+                    }
+                    returnData.push(obj)
+                }
+            }
+            return returnData
+        }
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const selectedStore = nodeData.inputs?.selectedStore as string
+        const appDataSource = options.appDataSource as DataSource
+        const databaseEntities = options.databaseEntities as IDatabaseEntity
+        const output = nodeData.outputs?.output as string
+
+        const entity = await appDataSource.getRepository(databaseEntities['DocumentStore']).findOneBy({ id: selectedStore })
+        if (!entity) {
+            return { error: 'Store not found' }
+        }
+        const data: ICommonObject = {}
+        data.output = output
+
+        // Prepare Embeddings Instance
+        const embeddingConfig = JSON.parse(entity.embeddingConfig)
+        data.embeddingName = embeddingConfig.name
+        data.embeddingConfig = embeddingConfig.config
+        let embeddingObj = await _createEmbeddingsObject(options.componentNodes, data, options)
+        if (!embeddingObj) {
+            return { error: 'Failed to create EmbeddingObj' }
+        }
+
+        // Prepare Vector Store Instance
+        const vsConfig = JSON.parse(entity.vectorStoreConfig)
+        data.vectorStoreName = vsConfig.name
+        data.vectorStoreConfig = vsConfig.config
+        if (data.inputs) {
+            data.vectorStoreConfig = { ...vsConfig.config, ...data.inputs }
+        }
+
+        // Prepare Vector Store Node Data
+        const vStoreNodeData = _createVectorStoreNodeData(options.componentNodes, data, embeddingObj)
+
+        // Finally create the Vector Store or Retriever object (data.output)
+        const vectorStoreObj = await _createVectorStoreObject(options.componentNodes, data)
+        const retrieverOrVectorStore = await vectorStoreObj.init(vStoreNodeData, '', options)
+        if (!retrieverOrVectorStore) {
+            return { error: 'Failed to create vectorStore' }
+        }
+        return retrieverOrVectorStore
+    }
+}
+
+const _createEmbeddingsObject = async (componentNodes: ICommonObject, data: ICommonObject, options: ICommonObject): Promise<any> => {
+    // prepare embedding node data
+    const embeddingComponent = componentNodes[data.embeddingName]
+    const embeddingNodeData: any = {
+        inputs: { ...data.embeddingConfig },
+        outputs: { output: 'document' },
+        id: `${embeddingComponent.name}_0`,
+        label: embeddingComponent.label,
+        name: embeddingComponent.name,
+        category: embeddingComponent.category,
+        inputParams: embeddingComponent.inputs || []
+    }
+    if (data.embeddingConfig.credential) {
+        embeddingNodeData.credential = data.embeddingConfig.credential
+    }
+
+    // init embedding object
+    const embeddingNodeInstanceFilePath = embeddingComponent.filePath as string
+    const embeddingNodeModule = await import(embeddingNodeInstanceFilePath)
+    const embeddingNodeInstance = new embeddingNodeModule.nodeClass()
+    return await embeddingNodeInstance.init(embeddingNodeData, '', options)
+}
+
+const _createVectorStoreNodeData = (componentNodes: ICommonObject, data: ICommonObject, embeddingObj: any) => {
+    const vectorStoreComponent = componentNodes[data.vectorStoreName]
+    const vStoreNodeData: any = {
+        id: `${vectorStoreComponent.name}_0`,
+        inputs: { ...data.vectorStoreConfig },
+        outputs: { output: data.output },
+        label: vectorStoreComponent.label,
+        name: vectorStoreComponent.name,
+        category: vectorStoreComponent.category
+    }
+    if (data.vectorStoreConfig.credential) {
+        vStoreNodeData.credential = data.vectorStoreConfig.credential
+    }
+
+    if (embeddingObj) {
+        vStoreNodeData.inputs.embeddings = embeddingObj
+    }
+
+    // Get all input params except the ones that are anchor points to avoid JSON stringify circular error
+    const filterInputParams = ['document', 'embeddings', 'recordManager']
+    const inputParams = vectorStoreComponent.inputs?.filter((input: any) => !filterInputParams.includes(input.name))
+    vStoreNodeData.inputParams = inputParams
+    return vStoreNodeData
+}
+
+const _createVectorStoreObject = async (componentNodes: ICommonObject, data: ICommonObject) => {
+    const vStoreNodeInstanceFilePath = componentNodes[data.vectorStoreName].filePath as string
+    const vStoreNodeModule = await import(vStoreNodeInstanceFilePath)
+    const vStoreNodeInstance = new vStoreNodeModule.nodeClass()
+    return vStoreNodeInstance
+}
+
+module.exports = { nodeClass: DocStore_VectorStores }

packages/components/nodes/vectorstores/DocumentStoreVS/dstore.svg

@@ -0,0 +1,15 @@
+<svg
+		xmlns="http://www.w3.org/2000/svg"
+		width="24"
+		height="24"
+		viewBox="0 0 24 24"
+		fill="none"
+		stroke="currentColor"
+		stroke-width="2"
+		stroke-linecap="round"
+		stroke-linejoin="round"
+>
+	<path d="M12 4l-8 4l8 4l8 -4l-8 -4" />
+	<path d="M4 12l8 4l8 -4" />
+	<path d="M4 16l8 4l8 -4" />
+</svg>

packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch.ts

@@ -163,6 +163,35 @@ class Elasticsearch_VectorStores implements INode {
             } catch (e) {
                 throw new Error(e)
             }
+        },
+        async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise<void> {
+            const indexName = nodeData.inputs?.indexName as string
+            const embeddings = nodeData.inputs?.embeddings as Embeddings
+            const similarityMeasure = nodeData.inputs?.similarityMeasure as string
+            const recordManager = nodeData.inputs?.recordManager
+
+            const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+            const endPoint = getCredentialParam('endpoint', credentialData, nodeData)
+            const cloudId = getCredentialParam('cloudId', credentialData, nodeData)
+
+            const elasticSearchClientArgs = prepareClientArgs(endPoint, cloudId, credentialData, nodeData, similarityMeasure, indexName)
+            const vectorStore = new ElasticVectorSearch(embeddings, elasticSearchClientArgs)
+
+            try {
+                if (recordManager) {
+                    const vectorStoreName = indexName
+                    await recordManager.createSchema()
+                    ;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
+                    const keys: string[] = await recordManager.listKeys({})
+
+                    await vectorStore.delete({ ids: keys })
+                    await recordManager.deleteKeys(keys)
+                } else {
+                    await vectorStore.delete({ ids })
+                }
+            } catch (e) {
+                throw new Error(e)
+            }
         }
     }
 

packages/components/nodes/vectorstores/Pinecone/Pinecone.ts

@@ -184,6 +184,45 @@ class Pinecone_VectorStores implements INode {
             } catch (e) {
                 throw new Error(e)
             }
+        },
+        async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise<void> {
+            const _index = nodeData.inputs?.pineconeIndex as string
+            const pineconeNamespace = nodeData.inputs?.pineconeNamespace as string
+            const embeddings = nodeData.inputs?.embeddings as Embeddings
+            const pineconeTextKey = nodeData.inputs?.pineconeTextKey as string
+            const recordManager = nodeData.inputs?.recordManager
+
+            const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+            const pineconeApiKey = getCredentialParam('pineconeApiKey', credentialData, nodeData)
+
+            const client = getPineconeClient({ apiKey: pineconeApiKey })
+
+            const pineconeIndex = client.Index(_index)
+
+            const obj: PineconeStoreParams = {
+                pineconeIndex,
+                textKey: pineconeTextKey || 'text'
+            }
+
+            if (pineconeNamespace) obj.namespace = pineconeNamespace
+            const pineconeStore = new PineconeStore(embeddings, obj)
+
+            try {
+                if (recordManager) {
+                    const vectorStoreName = pineconeNamespace
+                    await recordManager.createSchema()
+                    ;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
+                    const keys: string[] = await recordManager.listKeys({})
+
+                    await pineconeStore.delete({ ids: keys })
+                    await recordManager.deleteKeys(keys)
+                } else {
+                    const pineconeStore = new PineconeStore(embeddings, obj)
+                    await pineconeStore.delete({ ids })
+                }
+            } catch (e) {
+                throw new Error(e)
+            }
         }
     }
 

packages/components/nodes/vectorstores/Postgres/Postgres.ts

@@ -201,6 +201,58 @@ class Postgres_VectorStores implements INode {
             } catch (e) {
                 throw new Error(e)
             }
+        },
+        async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise<void> {
+            const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+            const user = getCredentialParam('user', credentialData, nodeData)
+            const password = getCredentialParam('password', credentialData, nodeData)
+            const _tableName = nodeData.inputs?.tableName as string
+            const tableName = _tableName ? _tableName : 'documents'
+            const embeddings = nodeData.inputs?.embeddings as Embeddings
+            const additionalConfig = nodeData.inputs?.additionalConfig as string
+            const recordManager = nodeData.inputs?.recordManager
+
+            let additionalConfiguration = {}
+            if (additionalConfig) {
+                try {
+                    additionalConfiguration = typeof additionalConfig === 'object' ? additionalConfig : JSON.parse(additionalConfig)
+                } catch (exception) {
+                    throw new Error('Invalid JSON in the Additional Configuration: ' + exception)
+                }
+            }
+
+            const postgresConnectionOptions = {
+                ...additionalConfiguration,
+                type: 'postgres',
+                host: nodeData.inputs?.host as string,
+                port: nodeData.inputs?.port as number,
+                username: user,
+                password: password,
+                database: nodeData.inputs?.database as string
+            }
+
+            const args = {
+                postgresConnectionOptions: postgresConnectionOptions as DataSourceOptions,
+                tableName: tableName
+            }
+
+            const vectorStore = await TypeORMVectorStore.fromDataSource(embeddings, args)
+
+            try {
+                if (recordManager) {
+                    const vectorStoreName = tableName
+                    await recordManager.createSchema()
+                    ;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
+                    const keys: string[] = await recordManager.listKeys({})
+
+                    await vectorStore.delete({ ids: keys })
+                    await recordManager.deleteKeys(keys)
+                } else {
+                    await vectorStore.delete({ ids })
+                }
+            } catch (e) {
+                throw new Error(e)
+            }
         }
     }
 

packages/components/nodes/vectorstores/Qdrant/Qdrant.ts

@@ -291,6 +291,69 @@ class Qdrant_VectorStores implements INode {
             } catch (e) {
                 throw new Error(e)
             }
+        },
+        async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise<void> {
+            const qdrantServerUrl = nodeData.inputs?.qdrantServerUrl as string
+            const collectionName = nodeData.inputs?.qdrantCollection as string
+            const embeddings = nodeData.inputs?.embeddings as Embeddings
+            const qdrantSimilarity = nodeData.inputs?.qdrantSimilarity
+            const qdrantVectorDimension = nodeData.inputs?.qdrantVectorDimension
+            const recordManager = nodeData.inputs?.recordManager
+
+            const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+            const qdrantApiKey = getCredentialParam('qdrantApiKey', credentialData, nodeData)
+
+            const port = Qdrant_VectorStores.determinePortByUrl(qdrantServerUrl)
+
+            const client = new QdrantClient({
+                url: qdrantServerUrl,
+                apiKey: qdrantApiKey,
+                port: port
+            })
+
+            const dbConfig: QdrantLibArgs = {
+                client,
+                url: qdrantServerUrl,
+                collectionName,
+                collectionConfig: {
+                    vectors: {
+                        size: qdrantVectorDimension ? parseInt(qdrantVectorDimension, 10) : 1536,
+                        distance: qdrantSimilarity ?? 'Cosine'
+                    }
+                }
+            }
+
+            const vectorStore = new QdrantVectorStore(embeddings, dbConfig)
+
+            vectorStore.delete = async (params: { ids: string[] }): Promise<void> => {
+                const { ids } = params
+
+                if (ids?.length) {
+                    try {
+                        client.delete(collectionName, {
+                            points: ids
+                        })
+                    } catch (e) {
+                        console.error('Failed to delete')
+                    }
+                }
+            }
+
+            try {
+                if (recordManager) {
+                    const vectorStoreName = collectionName
+                    await recordManager.createSchema()
+                    ;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
+                    const keys: string[] = await recordManager.listKeys({})
+
+                    await vectorStore.delete({ ids: keys })
+                    await recordManager.deleteKeys(keys)
+                } else {
+                    await vectorStore.delete({ ids })
+                }
+            } catch (e) {
+                throw new Error(e)
+            }
         }
     }
 

packages/components/nodes/vectorstores/Supabase/Supabase.ts

@@ -171,6 +171,40 @@ class Supabase_VectorStores implements INode {
             } catch (e) {
                 throw new Error(e)
             }
+        },
+        async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise<void> {
+            const supabaseProjUrl = nodeData.inputs?.supabaseProjUrl as string
+            const tableName = nodeData.inputs?.tableName as string
+            const queryName = nodeData.inputs?.queryName as string
+            const embeddings = nodeData.inputs?.embeddings as Embeddings
+            const recordManager = nodeData.inputs?.recordManager
+
+            const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+            const supabaseApiKey = getCredentialParam('supabaseApiKey', credentialData, nodeData)
+
+            const client = createClient(supabaseProjUrl, supabaseApiKey)
+
+            const supabaseStore = new SupabaseVectorStore(embeddings, {
+                client,
+                tableName: tableName,
+                queryName: queryName
+            })
+
+            try {
+                if (recordManager) {
+                    const vectorStoreName = tableName + '_' + queryName
+                    await recordManager.createSchema()
+                    ;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
+                    const keys: string[] = await recordManager.listKeys({})
+
+                    await supabaseStore.delete({ ids: keys })
+                    await recordManager.deleteKeys(keys)
+                } else {
+                    await supabaseStore.delete({ ids })
+                }
+            } catch (e) {
+                throw new Error(e)
+            }
         }
     }
 

packages/components/nodes/vectorstores/Upstash/Upstash.ts

@@ -145,6 +145,41 @@ class Upstash_VectorStores implements INode {
             } catch (e) {
                 throw new Error(e)
             }
+        },
+        async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise<void> {
+            const embeddings = nodeData.inputs?.embeddings as Embeddings
+            const recordManager = nodeData.inputs?.recordManager
+
+            const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+            const UPSTASH_VECTOR_REST_URL = getCredentialParam('UPSTASH_VECTOR_REST_URL', credentialData, nodeData)
+            const UPSTASH_VECTOR_REST_TOKEN = getCredentialParam('UPSTASH_VECTOR_REST_TOKEN', credentialData, nodeData)
+
+            const upstashIndex = new UpstashIndex({
+                url: UPSTASH_VECTOR_REST_URL,
+                token: UPSTASH_VECTOR_REST_TOKEN
+            })
+
+            const obj = {
+                index: upstashIndex
+            }
+
+            const upstashStore = new UpstashVectorStore(embeddings, obj)
+
+            try {
+                if (recordManager) {
+                    const vectorStoreName = UPSTASH_VECTOR_REST_URL
+                    await recordManager.createSchema()
+                    ;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
+                    const keys: string[] = await recordManager.listKeys({})
+
+                    await upstashStore.delete({ ids: keys })
+                    await recordManager.deleteKeys(keys)
+                } else {
+                    await upstashStore.delete({ ids })
+                }
+            } catch (e) {
+                throw new Error(e)
+            }
         }
     }
 

packages/components/nodes/vectorstores/Weaviate/Weaviate.ts

@@ -200,6 +200,53 @@ class Weaviate_VectorStores implements INode {
             } catch (e) {
                 throw new Error(e)
             }
+        },
+        async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise<void> {
+            const weaviateScheme = nodeData.inputs?.weaviateScheme as string
+            const weaviateHost = nodeData.inputs?.weaviateHost as string
+            const weaviateIndex = nodeData.inputs?.weaviateIndex as string
+            const weaviateTextKey = nodeData.inputs?.weaviateTextKey as string
+            const weaviateMetadataKeys = nodeData.inputs?.weaviateMetadataKeys as string
+            const embeddings = nodeData.inputs?.embeddings as Embeddings
+            const recordManager = nodeData.inputs?.recordManager
+
+            const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+            const weaviateApiKey = getCredentialParam('weaviateApiKey', credentialData, nodeData)
+
+            const clientConfig: any = {
+                scheme: weaviateScheme,
+                host: weaviateHost
+            }
+            if (weaviateApiKey) clientConfig.apiKey = new ApiKey(weaviateApiKey)
+
+            const client: WeaviateClient = weaviate.client(clientConfig)
+
+            const obj: WeaviateLibArgs = {
+                //@ts-ignore
+                client,
+                indexName: weaviateIndex
+            }
+
+            if (weaviateTextKey) obj.textKey = weaviateTextKey
+            if (weaviateMetadataKeys) obj.metadataKeys = JSON.parse(weaviateMetadataKeys.replace(/\s/g, ''))
+
+            const weaviateStore = new WeaviateStore(embeddings, obj)
+
+            try {
+                if (recordManager) {
+                    const vectorStoreName = weaviateTextKey ? weaviateIndex + '_' + weaviateTextKey : weaviateIndex
+                    await recordManager.createSchema()
+                    ;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
+                    const keys: string[] = await recordManager.listKeys({})
+
+                    await weaviateStore.delete({ ids: keys })
+                    await recordManager.deleteKeys(keys)
+                } else {
+                    await weaviateStore.delete({ ids })
+                }
+            } catch (e) {
+                throw new Error(e)
+            }
         }
     }
 

packages/components/src/Interface.ts

@@ -130,7 +130,7 @@ export interface INode extends INodeProperties {
     vectorStoreMethods?: {
         upsert: (nodeData: INodeData, options?: ICommonObject) => Promise<IndexingResult | void>
         search: (nodeData: INodeData, options?: ICommonObject) => Promise<any>
-        delete: (nodeData: INodeData, options?: ICommonObject) => Promise<void>
+        delete: (nodeData: INodeData, ids: string[], options?: ICommonObject) => Promise<void>
     }
     init?(nodeData: INodeData, input: string, options?: ICommonObject): Promise<any>
     run?(nodeData: INodeData, input: string, options?: ICommonObject): Promise<string | ICommonObject>

packages/server/src/Interface.DocumentStore.ts

@@ -5,7 +5,9 @@ export enum DocumentStoreStatus {
     SYNC = 'SYNC',
     SYNCING = 'SYNCING',
     STALE = 'STALE',
-    NEW = 'NEW'
+    NEW = 'NEW',
+    UPSERTING = 'UPSERTING',
+    UPSERTED = 'UPSERTED'
 }
 
 export interface IDocumentStore {
@@ -17,6 +19,9 @@ export interface IDocumentStore {
     updatedDate: Date
     createdDate: Date
     status: DocumentStoreStatus
+    vectorStoreConfig: string | null // JSON string
+    embeddingConfig: string | null // JSON string
+    recordManagerConfig: string | null // JSON string
 }
 
 export interface IDocumentStoreFileChunk {
@@ -89,6 +94,9 @@ export class DocumentStoreDTO {
     totalChars: number
     chunkSize: number
     loaders: IDocumentStoreLoader[]
+    vectorStoreConfig: any
+    embeddingConfig: any
+    recordManagerConfig: any
 
     constructor() {}
 
@@ -109,6 +117,16 @@ export class DocumentStoreDTO {
             documentStoreDTO.whereUsed = []
         }
 
+        if (entity.vectorStoreConfig) {
+            documentStoreDTO.vectorStoreConfig = JSON.parse(entity.vectorStoreConfig)
+        }
+        if (entity.embeddingConfig) {
+            documentStoreDTO.embeddingConfig = JSON.parse(entity.embeddingConfig)
+        }
+        if (entity.recordManagerConfig) {
+            documentStoreDTO.recordManagerConfig = JSON.parse(entity.recordManagerConfig)
+        }
+
         if (entity.loaders) {
             documentStoreDTO.loaders = JSON.parse(entity.loaders)
             documentStoreDTO.loaders.map((loader) => {

packages/server/src/controllers/documentstore/index.ts

@@ -248,6 +248,100 @@ const getDocumentLoaders = async (req: Request, res: Response, next: NextFunctio
     }
 }
 
+const insertIntoVectorStore = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        if (typeof req.body === 'undefined') {
+            throw new Error('Error: documentStoreController.insertIntoVectorStore - body not provided!')
+        }
+        const body = req.body
+        const apiResponse = await documentStoreService.insertIntoVectorStore(body)
+        return res.json(DocumentStoreDTO.fromEntity(apiResponse))
+    } catch (error) {
+        next(error)
+    }
+}
+
+const queryVectorStore = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        if (typeof req.body === 'undefined') {
+            throw new Error('Error: documentStoreController.queryVectorStore - body not provided!')
+        }
+        const body = req.body
+        const apiResponse = await documentStoreService.queryVectorStore(body)
+        return res.json(apiResponse)
+    } catch (error) {
+        next(error)
+    }
+}
+
+const deleteVectorStoreFromStore = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        if (typeof req.params.storeId === 'undefined' || req.params.storeId === '') {
+            throw new InternalFlowiseError(
+                StatusCodes.PRECONDITION_FAILED,
+                `Error: documentStoreController.deleteVectorStoreFromStore - storeId not provided!`
+            )
+        }
+        const apiResponse = await documentStoreService.deleteVectorStoreFromStore(req.params.storeId)
+        return res.json(apiResponse)
+    } catch (error) {
+        next(error)
+    }
+}
+
+const saveVectorStoreConfig = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        if (typeof req.body === 'undefined') {
+            throw new Error('Error: documentStoreController.saveVectorStoreConfig - body not provided!')
+        }
+        const body = req.body
+        const apiResponse = await documentStoreService.saveVectorStoreConfig(body)
+        return res.json(apiResponse)
+    } catch (error) {
+        next(error)
+    }
+}
+
+const updateVectorStoreConfigOnly = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        if (typeof req.body === 'undefined') {
+            throw new Error('Error: documentStoreController.updateVectorStoreConfigOnly - body not provided!')
+        }
+        const body = req.body
+        const apiResponse = await documentStoreService.updateVectorStoreConfigOnly(body)
+        return res.json(apiResponse)
+    } catch (error) {
+        next(error)
+    }
+}
+
+const getEmbeddingProviders = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        const apiResponse = await documentStoreService.getEmbeddingProviders()
+        return res.json(apiResponse)
+    } catch (error) {
+        next(error)
+    }
+}
+
+const getVectorStoreProviders = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        const apiResponse = await documentStoreService.getVectorStoreProviders()
+        return res.json(apiResponse)
+    } catch (error) {
+        next(error)
+    }
+}
+
+const getRecordManagerProviders = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        const apiResponse = await documentStoreService.getRecordManagerProviders()
+        return res.json(apiResponse)
+    } catch (error) {
+        next(error)
+    }
+}
+
 export default {
     deleteDocumentStore,
     createDocumentStore,
@@ -260,5 +354,13 @@ export default {
     previewFileChunks,
     getDocumentLoaders,
     deleteDocumentStoreFileChunk,
-    editDocumentStoreFileChunk
+    editDocumentStoreFileChunk,
+    insertIntoVectorStore,
+    getEmbeddingProviders,
+    getVectorStoreProviders,
+    getRecordManagerProviders,
+    saveVectorStoreConfig,
+    queryVectorStore,
+    deleteVectorStoreFromStore,
+    updateVectorStoreConfigOnly
 }

packages/server/src/database/entities/DocumentStore.ts

@@ -28,4 +28,13 @@ export class DocumentStore implements IDocumentStore {
 
     @Column({ nullable: false, type: 'text' })
     status: DocumentStoreStatus
+
+    @Column({ nullable: true, type: 'text' })
+    vectorStoreConfig: string | null
+
+    @Column({ nullable: true, type: 'text' })
+    embeddingConfig: string | null
+
+    @Column({ nullable: true, type: 'text' })
+    recordManagerConfig: string | null
 }

packages/server/src/database/migrations/mysql/1715861032479-AddVectorStoreConfigToDocStore.ts

@@ -0,0 +1,18 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddVectorStoreConfigToDocStore1715861032479 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        const columnExists = await queryRunner.hasColumn('document_store', 'vectorStoreConfig')
+        if (!columnExists) {
+            await queryRunner.query(`ALTER TABLE \`document_store\` ADD COLUMN \`vectorStoreConfig\` TEXT;`)
+            await queryRunner.query(`ALTER TABLE \`document_store\` ADD COLUMN \`embeddingConfig\` TEXT;`)
+            await queryRunner.query(`ALTER TABLE \`document_store\` ADD COLUMN \`recordManagerConfig\` TEXT;`)
+        }
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`document_store\` DROP COLUMN \`vectorStoreConfig\`;`)
+        await queryRunner.query(`ALTER TABLE \`document_store\` DROP COLUMN \`embeddingConfig\`;`)
+        await queryRunner.query(`ALTER TABLE \`document_store\` DROP COLUMN \`recordManagerConfig\`;`)
+    }
+}

packages/server/src/database/migrations/mysql/index.ts

@@ -17,6 +17,7 @@ import { AddFeedback1707213626553 } from './1707213626553-AddFeedback'
 import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHistoryEntity'
 import { AddLead1710832127079 } from './1710832127079-AddLead'
 import { AddLeadToChatMessage1711538023578 } from './1711538023578-AddLeadToChatMessage'
+import { AddVectorStoreConfigToDocStore1715861032479 } from './1715861032479-AddVectorStoreConfigToDocStore'
 import { AddDocumentStore1711637331047 } from './1711637331047-AddDocumentStore'
 import { AddAgentReasoningToChatMessage1714679514451 } from './1714679514451-AddAgentReasoningToChatMessage'
 import { AddTypeToChatFlow1716300000000 } from './1716300000000-AddTypeToChatFlow'
@@ -47,6 +48,7 @@ export const mysqlMigrations = [
     AddLeadToChatMessage1711538023578,
     AddAgentReasoningToChatMessage1714679514451,
     AddTypeToChatFlow1716300000000,
+    AddVectorStoreConfigToDocStore1715861032479,
     AddApiKey1720230151480,
     AddActionToChatMessage1721078251523,
     LongTextColumn1722301395521

packages/server/src/database/migrations/postgres/1715861032479-AddVectorStoreConfigToDocStore.ts

@@ -0,0 +1,15 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddVectorStoreConfigToDocStore1715861032479 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "document_store" ADD COLUMN IF NOT EXISTS "vectorStoreConfig" TEXT;`)
+        await queryRunner.query(`ALTER TABLE "document_store" ADD COLUMN IF NOT EXISTS "embeddingConfig" TEXT;`)
+        await queryRunner.query(`ALTER TABLE "document_store" ADD COLUMN IF NOT EXISTS "recordManagerConfig" TEXT;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "document_store" DROP COLUMN "vectorStoreConfig";`)
+        await queryRunner.query(`ALTER TABLE "document_store" DROP COLUMN "embeddingConfig";`)
+        await queryRunner.query(`ALTER TABLE "document_store" DROP COLUMN "recordManagerConfig";`)
+    }
+}

packages/server/src/database/migrations/postgres/index.ts

@@ -18,6 +18,7 @@ import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHi
 import { FieldTypes1710497452584 } from './1710497452584-FieldTypes'
 import { AddLead1710832137905 } from './1710832137905-AddLead'
 import { AddLeadToChatMessage1711538016098 } from './1711538016098-AddLeadToChatMessage'
+import { AddVectorStoreConfigToDocStore1715861032479 } from './1715861032479-AddVectorStoreConfigToDocStore'
 import { AddDocumentStore1711637331047 } from './1711637331047-AddDocumentStore'
 import { AddAgentReasoningToChatMessage1714679514451 } from './1714679514451-AddAgentReasoningToChatMessage'
 import { AddTypeToChatFlow1716300000000 } from './1716300000000-AddTypeToChatFlow'
@@ -48,6 +49,7 @@ export const postgresMigrations = [
     AddLeadToChatMessage1711538016098,
     AddAgentReasoningToChatMessage1714679514451,
     AddTypeToChatFlow1716300000000,
+    AddVectorStoreConfigToDocStore1715861032479,
     AddApiKey1720230151480,
     AddActionToChatMessage1721078251523
 ]

packages/server/src/database/migrations/sqlite/1715861032479-AddVectorStoreConfigToDocStore.ts

@@ -0,0 +1,15 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddVectorStoreConfigToDocStore1715861032479 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "document_store" ADD COLUMN "vectorStoreConfig" TEXT;`)
+        await queryRunner.query(`ALTER TABLE "document_store" ADD COLUMN "embeddingConfig" TEXT;`)
+        await queryRunner.query(`ALTER TABLE "document_store" ADD COLUMN "recordManagerConfig" TEXT;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "document_store" DROP COLUMN "vectorStoreConfig";`)
+        await queryRunner.query(`ALTER TABLE "document_store" DROP COLUMN "embeddingConfig";`)
+        await queryRunner.query(`ALTER TABLE "document_store" DROP COLUMN "recordManagerConfig";`)
+    }
+}

packages/server/src/database/migrations/sqlite/index.ts

@@ -17,6 +17,7 @@ import { AddFeedback1707213619308 } from './1707213619308-AddFeedback'
 import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHistoryEntity'
 import { AddLead1710832117612 } from './1710832117612-AddLead'
 import { AddLeadToChatMessage1711537986113 } from './1711537986113-AddLeadToChatMessage'
+import { AddVectorStoreConfigToDocStore1715861032479 } from './1715861032479-AddVectorStoreConfigToDocStore'
 import { AddDocumentStore1711637331047 } from './1711637331047-AddDocumentStore'
 import { AddAgentReasoningToChatMessage1714679514451 } from './1714679514451-AddAgentReasoningToChatMessage'
 import { AddTypeToChatFlow1716300000000 } from './1716300000000-AddTypeToChatFlow'
@@ -46,6 +47,7 @@ export const sqliteMigrations = [
     AddLeadToChatMessage1711537986113,
     AddAgentReasoningToChatMessage1714679514451,
     AddTypeToChatFlow1716300000000,
+    AddVectorStoreConfigToDocStore1715861032479,
     AddApiKey1720230151480,
     AddActionToChatMessage1721078251523
 ]

packages/server/src/routes/documentstore/index.ts

@@ -16,7 +16,7 @@ router.delete('/store/:id', documentStoreController.deleteDocumentStore)
 
 /** Component Nodes = Document Store - Loaders */
 // Get all loaders
-router.get('/loaders', documentStoreController.getDocumentLoaders)
+router.get('/components/loaders', documentStoreController.getDocumentLoaders)
 
 // delete loader from document store
 router.delete('/loader/:id/:loaderId', documentStoreController.deleteLoaderFromDocumentStore)
@@ -33,4 +33,22 @@ router.put('/chunks/:storeId/:loaderId/:chunkId', documentStoreController.editDo
 // Get all file chunks from the store
 router.get('/chunks/:storeId/:fileId/:pageNo', documentStoreController.getDocumentStoreFileChunks)
 
+// add chunks to the selected vector store
+router.post('/vectorstore/insert', documentStoreController.insertIntoVectorStore)
+// save the selected vector store
+router.post('/vectorstore/save', documentStoreController.saveVectorStoreConfig)
+// delete data from the selected vector store
+router.delete('/vectorstore/:storeId', documentStoreController.deleteVectorStoreFromStore)
+// query the vector store
+router.post('/vectorstore/query', documentStoreController.queryVectorStore)
+// Get all embedding providers
+router.get('/components/embeddings', documentStoreController.getEmbeddingProviders)
+// Get all vector store providers
+router.get('/components/vectorstore', documentStoreController.getVectorStoreProviders)
+// Get all Record Manager providers
+router.get('/components/recordmanager', documentStoreController.getRecordManagerProviders)
+
+// update the selected vector store from the playground
+router.post('/vectorstore/update', documentStoreController.updateVectorStoreConfigOnly)
+
 export default router

packages/server/src/services/documentstore/index.ts

@@ -1,6 +1,5 @@
 import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
 import { DocumentStore } from '../../database/entities/DocumentStore'
-// @ts-ignore
 import {
     addSingleFileToStorage,
     getFileFromStorage,
@@ -10,22 +9,29 @@ import {
     removeSpecificFileFromStorage
 } from 'flowise-components'
 import {
+    chatType,
     DocumentStoreStatus,
     IDocumentStoreFileChunkPagedResponse,
     IDocumentStoreLoader,
     IDocumentStoreLoaderFile,
     IDocumentStoreLoaderForPreview,
-    IDocumentStoreWhereUsed
+    IDocumentStoreWhereUsed,
+    INodeData
 } from '../../Interface'
 import { DocumentStoreFileChunk } from '../../database/entities/DocumentStoreFileChunk'
 import { v4 as uuidv4 } from 'uuid'
-import { databaseEntities } from '../../utils'
+import { databaseEntities, getAppVersion, saveUpsertFlowData } from '../../utils'
 import logger from '../../utils/logger'
 import nodesService from '../nodes'
 import { InternalFlowiseError } from '../../errors/internalFlowiseError'
 import { StatusCodes } from 'http-status-codes'
 import { getErrorMessage } from '../../errors/utils'
 import { ChatFlow } from '../../database/entities/ChatFlow'
+import { Document } from '@langchain/core/documents'
+import { App } from '../../index'
+import { UpsertHistory } from '../../database/entities/UpsertHistory'
+import { cloneDeep, omit } from 'lodash'
+import telemetryService from '../telemetry'
 
 const DOCUMENT_STORE_BASE_FOLDER = 'docustore'
 
@@ -234,8 +240,16 @@ const deleteDocumentStore = async (storeId: string) => {
         const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({
             id: storeId
         })
-        if (!entity) throw new Error(`Document store ${storeId} not found`)
+        if (!entity) {
+            throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document store ${storeId} not found`)
+        }
         await removeFilesFromStorage(DOCUMENT_STORE_BASE_FOLDER, entity.id)
+
+        // delete upsert history
+        await appServer.AppDataSource.getRepository(UpsertHistory).delete({
+            chatflowid: storeId
+        })
+
         // now delete the store
         const tbd = await appServer.AppDataSource.getRepository(DocumentStore).delete({
             id: storeId
@@ -285,6 +299,83 @@ const deleteDocumentStoreFileChunk = async (storeId: string, docId: string, chun
     }
 }
 
+const deleteVectorStoreFromStore = async (storeId: string) => {
+    try {
+        const appServer = getRunningExpressApp()
+        const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({
+            id: storeId
+        })
+        if (!entity) {
+            throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document store ${storeId} not found`)
+        }
+
+        if (!entity.embeddingConfig) {
+            throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Embedding for Document store ${storeId} not found`)
+        }
+
+        if (!entity.vectorStoreConfig) {
+            throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Vector Store for Document store ${storeId} not found`)
+        }
+
+        if (!entity.recordManagerConfig) {
+            throw new InternalFlowiseError(
+                StatusCodes.NOT_FOUND,
+                `Record Manager for Document Store ${storeId} is needed to delete data from Vector Store`
+            )
+        }
+
+        const options: ICommonObject = {
+            chatflowid: storeId,
+            appDataSource: appServer.AppDataSource,
+            databaseEntities,
+            logger
+        }
+
+        // Get Record Manager Instance
+        const recordManagerConfig = JSON.parse(entity.recordManagerConfig)
+        const recordManagerObj = await _createRecordManagerObject(
+            appServer,
+            { recordManagerName: recordManagerConfig.name, recordManagerConfig: recordManagerConfig.config },
+            options
+        )
+
+        // Get Embeddings Instance
+        const embeddingConfig = JSON.parse(entity.embeddingConfig)
+        const embeddingObj = await _createEmbeddingsObject(
+            appServer,
+            { embeddingName: embeddingConfig.name, embeddingConfig: embeddingConfig.config },
+            options
+        )
+
+        // Get Vector Store Node Data
+        const vectorStoreConfig = JSON.parse(entity.vectorStoreConfig)
+        const vStoreNodeData = _createVectorStoreNodeData(
+            appServer,
+            { vectorStoreName: vectorStoreConfig.name, vectorStoreConfig: vectorStoreConfig.config },
+            embeddingObj,
+            recordManagerObj
+        )
+
+        // Get Vector Store Instance
+        const vectorStoreObj = await _createVectorStoreObject(
+            appServer,
+            { vectorStoreName: vectorStoreConfig.name, vectorStoreConfig: vectorStoreConfig.config },
+            vStoreNodeData
+        )
+        const idsToDelete: string[] = [] // empty ids because we get it dynamically from the record manager
+
+        // Call the delete method of the vector store
+        if (vectorStoreObj.vectorStoreMethods.delete) {
+            await vectorStoreObj.vectorStoreMethods.delete(vStoreNodeData, idsToDelete, options)
+        }
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: documentStoreServices.deleteVectorStoreFromStore - ${getErrorMessage(error)}`
+        )
+    }
+}
+
 const editDocumentStoreFileChunk = async (storeId: string, docId: string, chunkId: string, content: string, metadata: ICommonObject) => {
     try {
         const appServer = getRunningExpressApp()
@@ -700,6 +791,417 @@ const updateDocumentStoreUsage = async (chatId: string, storeId: string | undefi
     }
 }
 
+const updateVectorStoreConfigOnly = async (data: ICommonObject) => {
+    try {
+        const appServer = getRunningExpressApp()
+        const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({
+            id: data.storeId
+        })
+        if (!entity) {
+            throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document store ${data.storeId} not found`)
+        }
+
+        if (data.vectorStoreName) {
+            entity.vectorStoreConfig = JSON.stringify({
+                config: data.vectorStoreConfig,
+                name: data.vectorStoreName
+            })
+
+            const updatedEntity = await appServer.AppDataSource.getRepository(DocumentStore).save(entity)
+            return updatedEntity
+        }
+        return {}
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: documentStoreServices.updateVectorStoreConfig - ${getErrorMessage(error)}`
+        )
+    }
+}
+const saveVectorStoreConfig = async (data: ICommonObject) => {
+    try {
+        const appServer = getRunningExpressApp()
+        const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({
+            id: data.storeId
+        })
+        if (!entity) {
+            throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document store ${data.storeId} not found`)
+        }
+
+        if (data.embeddingName) {
+            entity.embeddingConfig = JSON.stringify({
+                config: data.embeddingConfig,
+                name: data.embeddingName
+            })
+        } else if (!data.embeddingName && !data.embeddingConfig) {
+            entity.embeddingConfig = null
+        }
+
+        if (data.vectorStoreName) {
+            entity.vectorStoreConfig = JSON.stringify({
+                config: data.vectorStoreConfig,
+                name: data.vectorStoreName
+            })
+        } else if (!data.vectorStoreName && !data.vectorStoreConfig) {
+            entity.vectorStoreConfig = null
+        }
+
+        if (data.recordManagerName) {
+            entity.recordManagerConfig = JSON.stringify({
+                config: data.recordManagerConfig,
+                name: data.recordManagerName
+            })
+        } else if (!data.recordManagerName && !data.recordManagerConfig) {
+            entity.recordManagerConfig = null
+        }
+
+        if (entity.status !== DocumentStoreStatus.UPSERTED && (data.vectorStoreName || data.recordManagerName || data.embeddingName)) {
+            // if the store is not already in sync, mark it as sync
+            // this also means that the store is not yet sync'ed to vector store
+            entity.status = DocumentStoreStatus.SYNC
+        }
+        await appServer.AppDataSource.getRepository(DocumentStore).save(entity)
+        return entity
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: documentStoreServices.saveVectorStoreConfig - ${getErrorMessage(error)}`
+        )
+    }
+}
+
+const insertIntoVectorStore = async (data: ICommonObject) => {
+    try {
+        const appServer = getRunningExpressApp()
+        const entity = await saveVectorStoreConfig(data)
+        entity.status = DocumentStoreStatus.UPSERTING
+        await appServer.AppDataSource.getRepository(DocumentStore).save(entity)
+
+        // TODO: to be moved into a worker thread...
+        const indexResult = await _insertIntoVectorStoreWorkerThread(data)
+        return indexResult
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: documentStoreServices.insertIntoVectorStore - ${getErrorMessage(error)}`
+        )
+    }
+}
+
+const _insertIntoVectorStoreWorkerThread = async (data: ICommonObject) => {
+    try {
+        const appServer = getRunningExpressApp()
+        const entity = await saveVectorStoreConfig(data)
+        let upsertHistory: Record<string, any> = {}
+        const chatflowid = data.storeId // fake chatflowid because this is not tied to any chatflow
+
+        const options: ICommonObject = {
+            chatflowid,
+            appDataSource: appServer.AppDataSource,
+            databaseEntities,
+            logger
+        }
+
+        let recordManagerObj = undefined
+
+        // Get Record Manager Instance
+        if (data.recordManagerName && data.recordManagerConfig) {
+            recordManagerObj = await _createRecordManagerObject(appServer, data, options, upsertHistory)
+        }
+
+        // Get Embeddings Instance
+        const embeddingObj = await _createEmbeddingsObject(appServer, data, options, upsertHistory)
+
+        // Get Vector Store Node Data
+        const vStoreNodeData = _createVectorStoreNodeData(appServer, data, embeddingObj, recordManagerObj)
+        // Prepare docs for upserting
+        const chunks = await appServer.AppDataSource.getRepository(DocumentStoreFileChunk).find({
+            where: {
+                storeId: data.storeId
+            }
+        })
+        const docs: Document[] = chunks.map((chunk: DocumentStoreFileChunk) => {
+            return new Document({
+                pageContent: chunk.pageContent,
+                metadata: JSON.parse(chunk.metadata)
+            })
+        })
+        vStoreNodeData.inputs.document = docs
+
+        // Get Vector Store Instance
+        const vectorStoreObj = await _createVectorStoreObject(appServer, data, vStoreNodeData, upsertHistory)
+        const indexResult = await vectorStoreObj.vectorStoreMethods.upsert(vStoreNodeData, options)
+
+        // Save to DB
+        if (indexResult) {
+            const result = cloneDeep(upsertHistory)
+            result['flowData'] = JSON.stringify(result['flowData'])
+            result['result'] = JSON.stringify(omit(indexResult, ['totalKeys', 'addedDocs']))
+            result.chatflowid = chatflowid
+            const newUpsertHistory = new UpsertHistory()
+            Object.assign(newUpsertHistory, result)
+            const upsertHistoryItem = appServer.AppDataSource.getRepository(UpsertHistory).create(newUpsertHistory)
+            await appServer.AppDataSource.getRepository(UpsertHistory).save(upsertHistoryItem)
+        }
+
+        await telemetryService.createEvent({
+            name: `vector_upserted`,
+            data: {
+                version: await getAppVersion(),
+                chatlowId: chatflowid,
+                type: chatType.INTERNAL,
+                flowGraph: omit(indexResult['result'], ['totalKeys', 'addedDocs'])
+            }
+        })
+
+        entity.status = DocumentStoreStatus.UPSERTED
+        await appServer.AppDataSource.getRepository(DocumentStore).save(entity)
+
+        return indexResult ?? { result: 'Successfully Upserted' }
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: documentStoreServices._insertIntoVectorStoreWorkerThread - ${getErrorMessage(error)}`
+        )
+    }
+}
+
+// Get all component nodes - Embeddings
+const getEmbeddingProviders = async () => {
+    try {
+        const dbResponse = await nodesService.getAllNodesForCategory('Embeddings')
+        return dbResponse.filter((node) => !node.tags?.includes('LlamaIndex'))
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: documentStoreServices.getEmbeddingProviders - ${getErrorMessage(error)}`
+        )
+    }
+}
+
+// Get all component nodes - Vector Stores
+const getVectorStoreProviders = async () => {
+    try {
+        const dbResponse = await nodesService.getAllNodesForCategory('Vector Stores')
+        return dbResponse.filter((node) => !node.tags?.includes('LlamaIndex') && node.name !== 'documentStoreVS')
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: documentStoreServices.getVectorStoreProviders - ${getErrorMessage(error)}`
+        )
+    }
+}
+// Get all component nodes - Vector Stores
+const getRecordManagerProviders = async () => {
+    try {
+        const dbResponse = await nodesService.getAllNodesForCategory('Record Manager')
+        return dbResponse.filter((node) => !node.tags?.includes('LlamaIndex'))
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: documentStoreServices.getRecordManagerProviders - ${getErrorMessage(error)}`
+        )
+    }
+}
+
+const queryVectorStore = async (data: ICommonObject) => {
+    try {
+        const appServer = getRunningExpressApp()
+        const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({
+            id: data.storeId
+        })
+        if (!entity) {
+            throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Document store ${data.storeId} not found`)
+        }
+        const options: ICommonObject = {
+            chatflowid: uuidv4(),
+            appDataSource: appServer.AppDataSource,
+            databaseEntities,
+            logger
+        }
+
+        if (!entity.embeddingConfig) {
+            throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Embedding for ${data.storeId} is not configured`)
+        }
+
+        if (!entity.vectorStoreConfig) {
+            throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Vector Store for ${data.storeId} is not configured`)
+        }
+
+        const embeddingConfig = JSON.parse(entity.embeddingConfig)
+        data.embeddingName = embeddingConfig.name
+        data.embeddingConfig = embeddingConfig.config
+        let embeddingObj = await _createEmbeddingsObject(appServer, data, options)
+
+        const vsConfig = JSON.parse(entity.vectorStoreConfig)
+        data.vectorStoreName = vsConfig.name
+        data.vectorStoreConfig = vsConfig.config
+        if (data.inputs) {
+            data.vectorStoreConfig = { ...vsConfig.config, ...data.inputs }
+        }
+
+        const vStoreNodeData = _createVectorStoreNodeData(appServer, data, embeddingObj, undefined)
+
+        // Get Vector Store Instance
+        const vectorStoreObj = await _createVectorStoreObject(appServer, data, vStoreNodeData)
+        const retriever = await vectorStoreObj.init(vStoreNodeData, '', options)
+        if (!retriever) {
+            throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Failed to create retriever`)
+        }
+        const startMillis = Date.now()
+        const results = await retriever.invoke(data.query, undefined)
+        if (!results) {
+            throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Failed to retrieve results`)
+        }
+        const endMillis = Date.now()
+        const timeTaken = endMillis - startMillis
+        const docs: any = results.map((result: IDocument) => {
+            return {
+                pageContent: result.pageContent,
+                metadata: result.metadata,
+                id: uuidv4()
+            }
+        })
+        // query our document store chunk with the storeId and pageContent
+        for (const doc of docs) {
+            const documentStoreChunk = await appServer.AppDataSource.getRepository(DocumentStoreFileChunk).findOneBy({
+                storeId: data.storeId,
+                pageContent: doc.pageContent
+            })
+            if (documentStoreChunk) {
+                doc.id = documentStoreChunk.id
+                doc.chunkNo = documentStoreChunk.chunkNo
+            } else {
+                // this should not happen, only possible if the vector store has more content
+                // than our document store
+                doc.id = uuidv4()
+                doc.chunkNo = -1
+            }
+        }
+
+        return {
+            timeTaken: timeTaken,
+            docs: docs
+        }
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: documentStoreServices.queryVectorStore - ${getErrorMessage(error)}`
+        )
+    }
+}
+
+const _createEmbeddingsObject = async (
+    appServer: App,
+    data: ICommonObject,
+    options: ICommonObject,
+    upsertHistory?: Record<string, any>
+): Promise<any> => {
+    // prepare embedding node data
+    const embeddingComponent = appServer.nodesPool.componentNodes[data.embeddingName]
+    const embeddingNodeData: any = {
+        inputs: { ...data.embeddingConfig },
+        outputs: { output: 'document' },
+        id: `${embeddingComponent.name}_0`,
+        label: embeddingComponent.label,
+        name: embeddingComponent.name,
+        category: embeddingComponent.category,
+        inputParams: embeddingComponent.inputs || []
+    }
+    if (data.embeddingConfig.credential) {
+        embeddingNodeData.credential = data.embeddingConfig.credential
+    }
+
+    // save to upsert history
+    if (upsertHistory) upsertHistory['flowData'] = saveUpsertFlowData(embeddingNodeData, upsertHistory)
+
+    // init embedding object
+    const embeddingNodeInstanceFilePath = embeddingComponent.filePath as string
+    const embeddingNodeModule = await import(embeddingNodeInstanceFilePath)
+    const embeddingNodeInstance = new embeddingNodeModule.nodeClass()
+    const embeddingObj = await embeddingNodeInstance.init(embeddingNodeData, '', options)
+    if (!embeddingObj) {
+        throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Failed to create EmbeddingObj`)
+    }
+    return embeddingObj
+}
+
+const _createRecordManagerObject = async (
+    appServer: App,
+    data: ICommonObject,
+    options: ICommonObject,
+    upsertHistory?: Record<string, any>
+) => {
+    // prepare record manager node data
+    const recordManagerComponent = appServer.nodesPool.componentNodes[data.recordManagerName]
+    const rmNodeData: any = {
+        inputs: { ...data.recordManagerConfig },
+        id: `${recordManagerComponent.name}_0`,
+        inputParams: recordManagerComponent.inputs,
+        label: recordManagerComponent.label,
+        name: recordManagerComponent.name,
+        category: recordManagerComponent.category
+    }
+    if (data.recordManagerConfig.credential) {
+        rmNodeData.credential = data.recordManagerConfig.credential
+    }
+
+    // save to upsert history
+    if (upsertHistory) upsertHistory['flowData'] = saveUpsertFlowData(rmNodeData, upsertHistory)
+
+    // init record manager object
+    const rmNodeInstanceFilePath = recordManagerComponent.filePath as string
+    const rmNodeModule = await import(rmNodeInstanceFilePath)
+    const rmNodeInstance = new rmNodeModule.nodeClass()
+    const recordManagerObj = await rmNodeInstance.init(rmNodeData, '', options)
+    if (!recordManagerObj) {
+        throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Failed to create RecordManager obj`)
+    }
+    return recordManagerObj
+}
+
+const _createVectorStoreNodeData = (appServer: App, data: ICommonObject, embeddingObj: any, recordManagerObj?: any) => {
+    const vectorStoreComponent = appServer.nodesPool.componentNodes[data.vectorStoreName]
+    const vStoreNodeData: any = {
+        id: `${vectorStoreComponent.name}_0`,
+        inputs: { ...data.vectorStoreConfig },
+        outputs: { output: 'retriever' },
+        label: vectorStoreComponent.label,
+        name: vectorStoreComponent.name,
+        category: vectorStoreComponent.category
+    }
+    if (data.vectorStoreConfig.credential) {
+        vStoreNodeData.credential = data.vectorStoreConfig.credential
+    }
+
+    if (embeddingObj) {
+        vStoreNodeData.inputs.embeddings = embeddingObj
+    }
+
+    if (recordManagerObj) {
+        vStoreNodeData.inputs.recordManager = recordManagerObj
+    }
+
+    // Get all input params except the ones that are anchor points to avoid JSON stringify circular error
+    const filterInputParams = ['document', 'embeddings', 'recordManager']
+    const inputParams = vectorStoreComponent.inputs?.filter((input) => !filterInputParams.includes(input.name))
+    vStoreNodeData.inputParams = inputParams
+    return vStoreNodeData
+}
+
+const _createVectorStoreObject = async (
+    appServer: App,
+    data: ICommonObject,
+    vStoreNodeData: INodeData,
+    upsertHistory?: Record<string, any>
+) => {
+    const vStoreNodeInstanceFilePath = appServer.nodesPool.componentNodes[data.vectorStoreName].filePath as string
+    const vStoreNodeModule = await import(vStoreNodeInstanceFilePath)
+    const vStoreNodeInstance = new vStoreNodeModule.nodeClass()
+    if (upsertHistory) upsertHistory['flowData'] = saveUpsertFlowData(vStoreNodeData, upsertHistory)
+    return vStoreNodeInstance
+}
+
 export default {
     updateDocumentStoreUsage,
     deleteDocumentStore,
@@ -714,5 +1216,13 @@ export default {
     processAndSaveChunks,
     deleteDocumentStoreFileChunk,
     editDocumentStoreFileChunk,
-    getDocumentLoaders
+    getDocumentLoaders,
+    insertIntoVectorStore,
+    getEmbeddingProviders,
+    getVectorStoreProviders,
+    getRecordManagerProviders,
+    saveVectorStoreConfig,
+    queryVectorStore,
+    deleteVectorStoreFromStore,
+    updateVectorStoreConfigOnly
 }

packages/server/src/utils/index.ts

@@ -547,7 +547,8 @@ export const buildFlow = async ({
                     uploads,
                     baseURL,
                     socketIO,
-                    socketIOClientId
+                    socketIOClientId,
+                    componentNodes: componentNodes as ICommonObject
                 })
 
                 // Save dynamic variables

packages/ui/src/api/documentstore.js

@@ -1,7 +1,7 @@
 import client from './client'
 
 const getAllDocumentStores = () => client.get('/document-store/stores')
-const getDocumentLoaders = () => client.get('/document-store/loaders')
+const getDocumentLoaders = () => client.get('/document-store/components/loaders')
 const getSpecificDocumentStore = (id) => client.get(`/document-store/store/${id}`)
 const createDocumentStore = (body) => client.post(`/document-store/store`, body)
 const updateDocumentStore = (id, body) => client.put(`/document-store/store/${id}`, body)
@@ -16,6 +16,15 @@ const getFileChunks = (storeId, fileId, pageNo) => client.get(`/document-store/c
 const previewChunks = (body) => client.post('/document-store/loader/preview', body)
 const processChunks = (body) => client.post(`/document-store/loader/process`, body)
 
+const insertIntoVectorStore = (body) => client.post(`/document-store/vectorstore/insert`, body)
+const saveVectorStoreConfig = (body) => client.post(`/document-store/vectorstore/save`, body)
+const updateVectorStoreConfig = (body) => client.post(`/document-store/vectorstore/update`, body)
+const deleteVectorStoreDataFromStore = (storeId) => client.delete(`/document-store/vectorstore/${storeId}`)
+const queryVectorStore = (body) => client.post(`/document-store/vectorstore/query`, body)
+const getVectorStoreProviders = () => client.get('/document-store/components/vectorstore')
+const getEmbeddingProviders = () => client.get('/document-store/components/embeddings')
+const getRecordManagerProviders = () => client.get('/document-store/components/recordmanager')
+
 export default {
     getAllDocumentStores,
     getSpecificDocumentStore,
@@ -28,5 +37,13 @@ export default {
     getDocumentLoaders,
     deleteChunkFromStore,
     editChunkFromStore,
-    deleteDocumentStore
+    deleteDocumentStore,
+    insertIntoVectorStore,
+    getVectorStoreProviders,
+    getEmbeddingProviders,
+    getRecordManagerProviders,
+    saveVectorStoreConfig,
+    queryVectorStore,
+    deleteVectorStoreDataFromStore,
+    updateVectorStoreConfig
 }

packages/ui/src/routes/MainRoutes.jsx

@@ -33,6 +33,8 @@ const Documents = Loadable(lazy(() => import('@/views/docstore')))
 const DocumentStoreDetail = Loadable(lazy(() => import('@/views/docstore/DocumentStoreDetail')))
 const ShowStoredChunks = Loadable(lazy(() => import('@/views/docstore/ShowStoredChunks')))
 const LoaderConfigPreviewChunks = Loadable(lazy(() => import('@/views/docstore/LoaderConfigPreviewChunks')))
+const VectorStoreConfigure = Loadable(lazy(() => import('@/views/docstore/VectorStoreConfigure')))
+const VectorStoreQuery = Loadable(lazy(() => import('@/views/docstore/VectorStoreQuery')))
 
 // ==============================|| MAIN ROUTING ||============================== //
 
@@ -91,6 +93,14 @@ const MainRoutes = {
         {
             path: '/document-stores/:id/:name',
             element: <LoaderConfigPreviewChunks />
+        },
+        {
+            path: '/document-stores/vector/:id',
+            element: <VectorStoreConfigure />
+        },
+        {
+            path: '/document-stores/query/:id',
+            element: <VectorStoreQuery />
         }
     ]
 }

packages/ui/src/views/docstore/ComponentsListDialog.jsx

@@ -0,0 +1,190 @@
+import { useState, useEffect } from 'react'
+import { createPortal } from 'react-dom'
+import { useSelector, useDispatch } from 'react-redux'
+import PropTypes from 'prop-types'
+import { List, ListItemButton, Dialog, DialogContent, DialogTitle, Box, OutlinedInput, InputAdornment, Typography } from '@mui/material'
+import { useTheme } from '@mui/material/styles'
+import { IconSearch, IconX } from '@tabler/icons-react'
+
+// API
+
+// const
+import { baseURL } from '@/store/constant'
+import { HIDE_CANVAS_DIALOG, SHOW_CANVAS_DIALOG } from '@/store/actions'
+import useApi from '@/hooks/useApi'
+
+const ComponentsListDialog = ({ show, dialogProps, onCancel, apiCall, onSelected }) => {
+    const portalElement = document.getElementById('portal')
+    const customization = useSelector((state) => state.customization)
+    const dispatch = useDispatch()
+    const theme = useTheme()
+    const [searchValue, setSearchValue] = useState('')
+    const [provider, setProvider] = useState([])
+
+    const getProvidersApi = useApi(apiCall)
+
+    const onSearchChange = (val) => {
+        setSearchValue(val)
+    }
+
+    function filterFlows(data) {
+        return data?.name?.toLowerCase().indexOf(searchValue.toLowerCase()) > -1
+    }
+
+    useEffect(() => {
+        // if (dialogProps.embeddingsProvider) {
+        //     setProvider(dialogProps.provider)
+        // }
+    }, [dialogProps])
+
+    useEffect(() => {
+        getProvidersApi.request()
+
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [])
+
+    useEffect(() => {
+        if (getProvidersApi.data) {
+            setProvider(getProvidersApi.data)
+        }
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [getProvidersApi.data])
+
+    useEffect(() => {
+        if (show) dispatch({ type: SHOW_CANVAS_DIALOG })
+        else dispatch({ type: HIDE_CANVAS_DIALOG })
+        return () => dispatch({ type: HIDE_CANVAS_DIALOG })
+    }, [show, dispatch])
+
+    const component = show ? (
+        <Dialog
+            fullWidth
+            maxWidth='md'
+            open={show}
+            onClose={onCancel}
+            aria-labelledby='alert-dialog-title'
+            aria-describedby='alert-dialog-description'
+        >
+            <DialogTitle sx={{ fontSize: '1rem', p: 3, pb: 0 }} id='alert-dialog-title'>
+                {dialogProps.title}
+            </DialogTitle>
+            <DialogContent sx={{ display: 'flex', flexDirection: 'column', gap: 2, maxHeight: '75vh', position: 'relative', px: 3, pb: 3 }}>
+                <Box
+                    sx={{
+                        backgroundColor: customization.isDarkMode ? theme.palette.background.darkPaper : theme.palette.background.paper,
+                        pt: 2,
+                        position: 'sticky',
+                        top: 0,
+                        zIndex: 10
+                    }}
+                >
+                    <OutlinedInput
+                        sx={{ width: '100%', pr: 2, pl: 2, position: 'sticky' }}
+                        id='input-search-credential'
+                        value={searchValue}
+                        onChange={(e) => onSearchChange(e.target.value)}
+                        placeholder='Search'
+                        startAdornment={
+                            <InputAdornment position='start'>
+                                <IconSearch stroke={1.5} size='1rem' color={theme.palette.grey[500]} />
+                            </InputAdornment>
+                        }
+                        endAdornment={
+                            <InputAdornment
+                                position='end'
+                                sx={{
+                                    cursor: 'pointer',
+                                    color: theme.palette.grey[500],
+                                    '&:hover': {
+                                        color: theme.palette.grey[900]
+                                    }
+                                }}
+                                title='Clear Search'
+                            >
+                                <IconX
+                                    stroke={1.5}
+                                    size='1rem'
+                                    onClick={() => onSearchChange('')}
+                                    style={{
+                                        cursor: 'pointer'
+                                    }}
+                                />
+                            </InputAdornment>
+                        }
+                        aria-describedby='search-helper-text'
+                        inputProps={{
+                            'aria-label': 'weight'
+                        }}
+                    />
+                </Box>
+                <List
+                    sx={{
+                        width: '100%',
+                        display: 'grid',
+                        gridTemplateColumns: 'repeat(3, 1fr)',
+                        gap: 2,
+                        py: 0,
+                        zIndex: 9,
+                        borderRadius: '10px',
+                        [theme.breakpoints.down('md')]: {
+                            maxWidth: 370
+                        }
+                    }}
+                >
+                    {[...provider].filter(filterFlows).map((loader) => (
+                        <ListItemButton
+                            alignItems='center'
+                            key={loader.name}
+                            onClick={() => onSelected(loader)}
+                            sx={{
+                                border: 1,
+                                borderColor: theme.palette.grey[900] + 25,
+                                borderRadius: 2,
+                                display: 'flex',
+                                alignItems: 'center',
+                                justifyContent: 'start',
+                                textAlign: 'left',
+                                gap: 1,
+                                p: 2
+                            }}
+                        >
+                            <div
+                                style={{
+                                    width: 50,
+                                    height: 50,
+                                    borderRadius: '50%',
+                                    backgroundColor: 'white'
+                                }}
+                            >
+                                <img
+                                    style={{
+                                        width: '100%',
+                                        height: '100%',
+                                        padding: 7,
+                                        borderRadius: '50%',
+                                        objectFit: 'contain'
+                                    }}
+                                    alt={loader.name}
+                                    src={`${baseURL}/api/v1/node-icon/${loader.name}`}
+                                />
+                            </div>
+                            <Typography>{loader.label}</Typography>
+                        </ListItemButton>
+                    ))}
+                </List>
+            </DialogContent>
+        </Dialog>
+    ) : null
+
+    return createPortal(component, portalElement)
+}
+
+ComponentsListDialog.propTypes = {
+    show: PropTypes.bool,
+    dialogProps: PropTypes.object,
+    onCancel: PropTypes.func,
+    apiCall: PropTypes.func,
+    onSelected: PropTypes.func
+}
+
+export default ComponentsListDialog

packages/ui/src/views/docstore/DeleteDocStoreDialog.jsx

@@ -0,0 +1,246 @@
+import { useState, useEffect } from 'react'
+import { createPortal } from 'react-dom'
+import PropTypes from 'prop-types'
+import { cloneDeep } from 'lodash'
+import {
+    Button,
+    Box,
+    Paper,
+    Accordion,
+    AccordionSummary,
+    AccordionDetails,
+    Dialog,
+    DialogContent,
+    DialogTitle,
+    Typography,
+    Table,
+    TableBody,
+    TableContainer,
+    TableRow,
+    TableCell,
+    Checkbox,
+    FormControlLabel,
+    DialogActions
+} from '@mui/material'
+import ExpandMoreIcon from '@mui/icons-material/ExpandMore'
+import { TableViewOnly } from '@/ui-component/table/Table'
+import { v4 as uuidv4 } from 'uuid'
+
+// const
+import { baseURL } from '@/store/constant'
+import nodesApi from '@/api/nodes'
+
+// Hooks
+import useApi from '@/hooks/useApi'
+import { initNode } from '@/utils/genericHelper'
+
+const DeleteDocStoreDialog = ({ show, dialogProps, onCancel, onDelete }) => {
+    const portalElement = document.getElementById('portal')
+    const [nodeConfigExpanded, setNodeConfigExpanded] = useState({})
+    const [removeFromVS, setRemoveFromVS] = useState(false)
+    const [vsFlowData, setVSFlowData] = useState([])
+    const [rmFlowData, setRMFlowData] = useState([])
+
+    const getSpecificNodeApi = useApi(nodesApi.getSpecificNode)
+
+    const handleAccordionChange = (nodeName) => (event, isExpanded) => {
+        const accordianNodes = { ...nodeConfigExpanded }
+        accordianNodes[nodeName] = isExpanded
+        setNodeConfigExpanded(accordianNodes)
+    }
+
+    useEffect(() => {
+        if (dialogProps.recordManagerConfig) {
+            const nodeName = dialogProps.recordManagerConfig.name
+            if (nodeName) getSpecificNodeApi.request(nodeName)
+
+            if (dialogProps.vectorStoreConfig) {
+                const nodeName = dialogProps.vectorStoreConfig.name
+                if (nodeName) getSpecificNodeApi.request(nodeName)
+            }
+        }
+
+        return () => {
+            setNodeConfigExpanded({})
+            setRemoveFromVS(false)
+            setVSFlowData([])
+            setRMFlowData([])
+        }
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [dialogProps])
+
+    useEffect(() => {
+        if (getSpecificNodeApi.data) {
+            const nodeData = cloneDeep(initNode(getSpecificNodeApi.data, uuidv4()))
+
+            let config = 'vectorStoreConfig'
+            if (nodeData.category === 'Record Manager') config = 'recordManagerConfig'
+
+            const paramValues = []
+
+            for (const inputName in dialogProps[config].config) {
+                const inputParam = nodeData.inputParams.find((inp) => inp.name === inputName)
+
+                if (!inputParam) continue
+
+                if (inputParam.type === 'credential') continue
+
+                let paramValue = {}
+
+                const inputValue = dialogProps[config].config[inputName]
+
+                if (!inputValue) continue
+
+                if (typeof inputValue === 'string' && inputValue.startsWith('{{') && inputValue.endsWith('}}')) {
+                    continue
+                }
+
+                paramValue = {
+                    label: inputParam?.label,
+                    name: inputParam?.name,
+                    type: inputParam?.type,
+                    value: inputValue
+                }
+                paramValues.push(paramValue)
+            }
+
+            if (config === 'vectorStoreConfig') {
+                setVSFlowData([
+                    {
+                        label: nodeData.label,
+                        name: nodeData.name,
+                        category: nodeData.category,
+                        id: nodeData.id,
+                        paramValues
+                    }
+                ])
+            } else if (config === 'recordManagerConfig') {
+                setRMFlowData([
+                    {
+                        label: nodeData.label,
+                        name: nodeData.name,
+                        category: nodeData.category,
+                        id: nodeData.id,
+                        paramValues
+                    }
+                ])
+            }
+        }
+
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [getSpecificNodeApi.data])
+
+    const component = show ? (
+        <Dialog
+            fullWidth
+            maxWidth={dialogProps.recordManagerConfig ? 'md' : 'sm'}
+            open={show}
+            onClose={onCancel}
+            aria-labelledby='alert-dialog-title'
+            aria-describedby='alert-dialog-description'
+        >
+            <DialogTitle sx={{ fontSize: '1rem', p: 3, pb: 0 }} id='alert-dialog-title'>
+                {dialogProps.title}
+            </DialogTitle>
+            <DialogContent sx={{ display: 'flex', flexDirection: 'column', gap: 2, maxHeight: '75vh', position: 'relative', px: 3, pb: 3 }}>
+                <span style={{ marginTop: '20px' }}>{dialogProps.description}</span>
+                {dialogProps.type === 'STORE' && dialogProps.recordManagerConfig && (
+                    <FormControlLabel
+                        control={<Checkbox checked={removeFromVS} onChange={(event) => setRemoveFromVS(event.target.checked)} />}
+                        label='Remove data from vector store'
+                    />
+                )}
+                {removeFromVS && (
+                    <div>
+                        <TableContainer component={Paper}>
+                            <Table sx={{ minWidth: 650 }} aria-label='simple table'>
+                                <TableBody>
+                                    <TableRow sx={{ '& td': { border: 0 } }}>
+                                        <TableCell sx={{ pb: 0, pt: 0 }} colSpan={6}>
+                                            <Box>
+                                                {([...vsFlowData, ...rmFlowData] || []).map((node, index) => {
+                                                    return (
+                                                        <Accordion
+                                                            expanded={nodeConfigExpanded[node.name] || true}
+                                                            onChange={handleAccordionChange(node.name)}
+                                                            key={index}
+                                                            disableGutters
+                                                        >
+                                                            <AccordionSummary
+                                                                expandIcon={<ExpandMoreIcon />}
+                                                                aria-controls={`nodes-accordian-${node.name}`}
+                                                                id={`nodes-accordian-header-${node.name}`}
+                                                            >
+                                                                <div
+                                                                    style={{ display: 'flex', flexDirection: 'row', alignItems: 'center' }}
+                                                                >
+                                                                    <div
+                                                                        style={{
+                                                                            width: 40,
+                                                                            height: 40,
+                                                                            marginRight: 10,
+                                                                            borderRadius: '50%',
+                                                                            backgroundColor: 'white'
+                                                                        }}
+                                                                    >
+                                                                        <img
+                                                                            style={{
+                                                                                width: '100%',
+                                                                                height: '100%',
+                                                                                padding: 7,
+                                                                                borderRadius: '50%',
+                                                                                objectFit: 'contain'
+                                                                            }}
+                                                                            alt={node.name}
+                                                                            src={`${baseURL}/api/v1/node-icon/${node.name}`}
+                                                                        />
+                                                                    </div>
+                                                                    <Typography variant='h5'>{node.label}</Typography>
+                                                                </div>
+                                                            </AccordionSummary>
+                                                            <AccordionDetails>
+                                                                {node.paramValues[0] && (
+                                                                    <TableViewOnly
+                                                                        sx={{ minWidth: 150 }}
+                                                                        rows={node.paramValues}
+                                                                        columns={Object.keys(node.paramValues[0])}
+                                                                    />
+                                                                )}
+                                                            </AccordionDetails>
+                                                        </Accordion>
+                                                    )
+                                                })}
+                                            </Box>
+                                        </TableCell>
+                                    </TableRow>
+                                </TableBody>
+                            </Table>
+                        </TableContainer>
+                        <span style={{ marginTop: '30px', fontStyle: 'italic', color: '#b35702' }}>
+                            * Only data that were upserted with Record Manager will be deleted from vector store
+                        </span>
+                    </div>
+                )}
+            </DialogContent>
+            <DialogActions sx={{ pr: 3, pb: 3 }}>
+                <Button onClick={onCancel} color='primary'>
+                    Cancel
+                </Button>
+                <Button variant='contained' onClick={() => onDelete(dialogProps.type, removeFromVS)} color='error'>
+                    Delete
+                </Button>
+            </DialogActions>
+        </Dialog>
+    ) : null
+
+    return createPortal(component, portalElement)
+}
+
+DeleteDocStoreDialog.propTypes = {
+    show: PropTypes.bool,
+    dialogProps: PropTypes.object,
+    onCancel: PropTypes.func,
+    onDelete: PropTypes.func
+}
+
+export default DeleteDocStoreDialog

packages/ui/src/views/docstore/DocStoreInputHandler.jsx

@@ -118,8 +118,9 @@ const DocStoreInputHandler = ({ inputParam, data, disabled = false }) => {
                         )}
                         {inputParam.type === 'credential' && (
                             <CredentialInputHandler
+                                key={JSON.stringify(inputParam)}
                                 disabled={disabled}
-                                data={data}
+                                data={data.inputs.credential ? { credential: data.inputs.credential } : {}}
                                 inputParam={inputParam}
                                 onSelect={(newValue) => {
                                     data.credential = newValue
@@ -189,6 +190,7 @@ const DocStoreInputHandler = ({ inputParam, data, disabled = false }) => {
                         )}
                         {inputParam.type === 'options' && (
                             <Dropdown
+                                key={JSON.stringify(inputParam)}
                                 disabled={disabled}
                                 name={inputParam.name}
                                 options={inputParam.options}
@@ -198,6 +200,7 @@ const DocStoreInputHandler = ({ inputParam, data, disabled = false }) => {
                         )}
                         {inputParam.type === 'multiOptions' && (
                             <MultiDropdown
+                                key={JSON.stringify(inputParam)}
                                 disabled={disabled}
                                 name={inputParam.name}
                                 options={inputParam.options}
@@ -207,9 +210,10 @@ const DocStoreInputHandler = ({ inputParam, data, disabled = false }) => {
                         )}
                         {inputParam.type === 'asyncOptions' && (
                             <>
-                                {data.inputParams.length === 1 && <div style={{ marginTop: 10 }} />}
+                                {data.inputParams?.length === 1 && <div style={{ marginTop: 10 }} />}
                                 <div style={{ display: 'flex', flexDirection: 'row' }}>
                                     <AsyncDropdown
+                                        key={JSON.stringify(inputParam)}
                                         disabled={disabled}
                                         name={inputParam.name}
                                         nodeData={data}

packages/ui/src/views/docstore/DocumentStoreDetail.jsx

@@ -29,20 +29,31 @@ import { tableCellClasses } from '@mui/material/TableCell'
 // project imports
 import MainCard from '@/ui-component/cards/MainCard'
 import AddDocStoreDialog from '@/views/docstore/AddDocStoreDialog'
-import ConfirmDialog from '@/ui-component/dialog/ConfirmDialog'
+import { BackdropLoader } from '@/ui-component/loading/BackdropLoader'
 import DocumentLoaderListDialog from '@/views/docstore/DocumentLoaderListDialog'
 import ErrorBoundary from '@/ErrorBoundary'
+import { StyledButton } from '@/ui-component/button/StyledButton'
+import ViewHeader from '@/layout/MainLayout/ViewHeader'
+import DeleteDocStoreDialog from './DeleteDocStoreDialog'
 
 // API
 import documentsApi from '@/api/documentstore'
 
 // Hooks
 import useApi from '@/hooks/useApi'
-import useConfirm from '@/hooks/useConfirm'
 import useNotifier from '@/utils/useNotifier'
 
 // icons
-import { IconPlus, IconRefresh, IconScissors, IconTrash, IconX, IconVectorBezier2 } from '@tabler/icons-react'
+import {
+    IconPlus,
+    IconRefresh,
+    IconListDetails,
+    IconTrash,
+    IconX,
+    IconVectorBezier2,
+    IconRowInsertTop,
+    IconZoomScan
+} from '@tabler/icons-react'
 import KeyboardArrowDownIcon from '@mui/icons-material/KeyboardArrowDown'
 import FileDeleteIcon from '@mui/icons-material/Delete'
 import FileEditIcon from '@mui/icons-material/Edit'
@@ -51,8 +62,6 @@ import doc_store_details_emptySVG from '@/assets/images/doc_store_details_empty.
 
 // store
 import { closeSnackbar as closeSnackbarAction, enqueueSnackbar as enqueueSnackbarAction } from '@/store/actions'
-import { StyledButton } from '@/ui-component/button/StyledButton'
-import ViewHeader from '@/layout/MainLayout/ViewHeader'
 
 // ==============================|| DOCUMENTS ||============================== //
 
@@ -121,17 +130,19 @@ const DocumentStoreDetails = () => {
 
     const enqueueSnackbar = (...args) => dispatch(enqueueSnackbarAction(...args))
     const closeSnackbar = (...args) => dispatch(closeSnackbarAction(...args))
-    const { confirm } = useConfirm()
 
     const getSpecificDocumentStore = useApi(documentsApi.getSpecificDocumentStore)
 
     const [error, setError] = useState(null)
     const [isLoading, setLoading] = useState(true)
+    const [isBackdropLoading, setBackdropLoading] = useState(false)
     const [showDialog, setShowDialog] = useState(false)
     const [documentStore, setDocumentStore] = useState({})
     const [dialogProps, setDialogProps] = useState({})
     const [showDocumentLoaderListDialog, setShowDocumentLoaderListDialog] = useState(false)
     const [documentLoaderListDialogProps, setDocumentLoaderListDialogProps] = useState({})
+    const [showDeleteDocStoreDialog, setShowDeleteDocStoreDialog] = useState(false)
+    const [deleteDocStoreDialogProps, setDeleteDocStoreDialogProps] = useState({})
 
     const URLpath = document.location.pathname.toString().split('/')
     const storeId = URLpath[URLpath.length - 1] === 'document-stores' ? '' : URLpath[URLpath.length - 1]
@@ -144,11 +155,19 @@ const DocumentStoreDetails = () => {
         navigate('/document-stores/chunks/' + storeId + '/' + id)
     }
 
+    const showVectorStoreQuery = (id) => {
+        navigate('/document-stores/query/' + id)
+    }
+
     const onDocLoaderSelected = (docLoaderComponentName) => {
         setShowDocumentLoaderListDialog(false)
         navigate('/document-stores/' + storeId + '/' + docLoaderComponentName)
     }
 
+    const showVectorStore = (id) => {
+        navigate('/document-stores/vector/' + id)
+    }
+
     const listLoaders = () => {
         const dialogProp = {
             title: 'Select Document Loader'
@@ -157,18 +176,60 @@ const DocumentStoreDetails = () => {
         setShowDocumentLoaderListDialog(true)
     }
 
-    const onLoaderDelete = async (file) => {
+    const deleteVectorStoreDataFromStore = async (storeId) => {
-        const confirmPayload = {
+        try {
-            title: `Delete`,
+            await documentsApi.deleteVectorStoreDataFromStore(storeId)
-            description: `Delete Loader ${file.loaderName} ? This will delete all the associated document chunks.`,
+        } catch (error) {
-            confirmButtonName: 'Delete',
+            console.error(error)
-            cancelButtonName: 'Cancel'
         }
-        const isConfirmed = await confirm(confirmPayload)
+    }
 
-        if (isConfirmed) {
+    const onDocStoreDelete = async (type, removeFromVectorStore) => {
+        setBackdropLoading(true)
+        if (type === 'STORE') {
+            if (removeFromVectorStore) {
+                await deleteVectorStoreDataFromStore(storeId)
+            }
+            try {
+                const deleteResp = await documentsApi.deleteDocumentStore(storeId)
+                setBackdropLoading(false)
+                if (deleteResp.data) {
+                    enqueueSnackbar({
+                        message: 'Store, Loader and associated document chunks deleted',
+                        options: {
+                            key: new Date().getTime() + Math.random(),
+                            variant: 'success',
+                            action: (key) => (
+                                <Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
+                                    <IconX />
+                                </Button>
+                            )
+                        }
+                    })
+                    navigate('/document-stores/')
+                }
+            } catch (error) {
+                setBackdropLoading(false)
+                setError(error)
+                const errorData = error.response.data || `${error.response.status}: ${error.response.statusText}`
+                enqueueSnackbar({
+                    message: `Failed to delete loader: ${errorData}`,
+                    options: {
+                        key: new Date().getTime() + Math.random(),
+                        variant: 'error',
+                        persist: true,
+                        action: (key) => (
+                            <Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
+                                <IconX />
+                            </Button>
+                        )
+                    }
+                })
+            }
+        } else if (type === 'LOADER') {
             try {
                 const deleteResp = await documentsApi.deleteLoaderFromStore(storeId, file.id)
+                setBackdropLoading(false)
                 if (deleteResp.data) {
                     enqueueSnackbar({
                         message: 'Loader and associated document chunks deleted',
@@ -186,6 +247,7 @@ const DocumentStoreDetails = () => {
                 }
             } catch (error) {
                 setError(error)
+                setBackdropLoading(false)
                 const errorData = error.response.data || `${error.response.status}: ${error.response.statusText}`
                 enqueueSnackbar({
                     message: `Failed to delete loader: ${errorData}`,
@@ -204,51 +266,30 @@ const DocumentStoreDetails = () => {
         }
     }
 
-    const onStoreDelete = async () => {
+    const onLoaderDelete = (file, vectorStoreConfig, recordManagerConfig) => {
-        const confirmPayload = {
+        const props = {
+            title: `Delete`,
+            description: `Delete Loader ${file.loaderName} ? This will delete all the associated document chunks.`,
+            vectorStoreConfig,
+            recordManagerConfig,
+            type: 'LOADER'
+        }
+
+        setDeleteDocStoreDialogProps(props)
+        setShowDeleteDocStoreDialog(true)
+    }
+
+    const onStoreDelete = (vectorStoreConfig, recordManagerConfig) => {
+        const props = {
             title: `Delete`,
             description: `Delete Store ${getSpecificDocumentStore.data?.name} ? This will delete all the associated loaders and document chunks.`,
-            confirmButtonName: 'Delete',
+            vectorStoreConfig,
-            cancelButtonName: 'Cancel'
+            recordManagerConfig,
+            type: 'STORE'
         }
-        const isConfirmed = await confirm(confirmPayload)
 
-        if (isConfirmed) {
+        setDeleteDocStoreDialogProps(props)
-            try {
+        setShowDeleteDocStoreDialog(true)
-                const deleteResp = await documentsApi.deleteDocumentStore(storeId)
-                if (deleteResp.data) {
-                    enqueueSnackbar({
-                        message: 'Store, Loader and associated document chunks deleted',
-                        options: {
-                            key: new Date().getTime() + Math.random(),
-                            variant: 'success',
-                            action: (key) => (
-                                <Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
-                                    <IconX />
-                                </Button>
-                            )
-                        }
-                    })
-                    navigate('/document-stores/')
-                }
-            } catch (error) {
-                setError(error)
-                const errorData = error.response.data || `${error.response.status}: ${error.response.statusText}`
-                enqueueSnackbar({
-                    message: `Failed to delete loader: ${errorData}`,
-                    options: {
-                        key: new Date().getTime() + Math.random(),
-                        variant: 'error',
-                        persist: true,
-                        action: (key) => (
-                            <Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
-                                <IconX />
-                            </Button>
-                        )
-                    }
-                })
-            }
-        }
     }
 
     const onEditClicked = () => {
@@ -315,24 +356,15 @@ const DocumentStoreDetails = () => {
                             onBack={() => navigate('/document-stores')}
                             onEdit={() => onEditClicked()}
                         >
-                            <IconButton onClick={onStoreDelete} size='small' color='error' title='Delete Document Store' sx={{ mr: 2 }}>
+                            <IconButton
+                                onClick={() => onStoreDelete(documentStore.vectorStoreConfig, documentStore.recordManagerConfig)}
+                                size='small'
+                                color='error'
+                                title='Delete Document Store'
+                                sx={{ mr: 2 }}
+                            >
                                 <IconTrash />
                             </IconButton>
-                            {documentStore?.status === 'STALE' && (
-                                <Button variant='outlined' sx={{ mr: 2 }} startIcon={<IconRefresh />} onClick={onConfirm}>
-                                    Refresh
-                                </Button>
-                            )}
-                            {documentStore?.totalChunks > 0 && (
-                                <Button
-                                    variant='outlined'
-                                    sx={{ borderRadius: 2, height: '100%' }}
-                                    startIcon={<IconScissors />}
-                                    onClick={() => showStoredChunks('all')}
-                                >
-                                    View Chunks
-                                </Button>
-                            )}
                             <StyledButton
                                 variant='contained'
                                 sx={{ borderRadius: 2, height: '100%', color: 'white' }}
@@ -341,6 +373,67 @@ const DocumentStoreDetails = () => {
                             >
                                 Add Document Loader
                             </StyledButton>
+                            {(documentStore?.status === 'STALE' || documentStore?.status === 'UPSERTING') && (
+                                <Button variant='outlined' sx={{ mr: 2 }} startIcon={<IconRefresh />} onClick={onConfirm}>
+                                    Refresh
+                                </Button>
+                            )}
+                            {documentStore?.status === 'UPSERTING' && (
+                                <Chip
+                                    variant='raised'
+                                    label='Upserting to Vector Store'
+                                    color='warning'
+                                    sx={{ borderRadius: 2, height: '100%' }}
+                                />
+                            )}
+                            {documentStore?.totalChunks > 0 && documentStore?.status !== 'UPSERTING' && (
+                                <>
+                                    <Button
+                                        variant='contained'
+                                        sx={{
+                                            borderRadius: 2,
+                                            height: '100%'
+                                        }}
+                                        color='secondary'
+                                        startIcon={<IconListDetails />}
+                                        onClick={() => showStoredChunks('all')}
+                                    >
+                                        View Chunks
+                                    </Button>
+                                    <Button
+                                        variant='contained'
+                                        sx={{
+                                            borderRadius: 2,
+                                            height: '100%',
+                                            backgroundImage: `linear-gradient(to right, #13547a, #2f9e91)`,
+                                            '&:hover': {
+                                                backgroundImage: `linear-gradient(to right, #0b3d5b, #1a8377)`
+                                            }
+                                        }}
+                                        startIcon={<IconRowInsertTop />}
+                                        onClick={() => showVectorStore(documentStore.id)}
+                                    >
+                                        Upsert Config
+                                    </Button>
+                                </>
+                            )}
+                            {documentStore?.totalChunks > 0 && documentStore?.status === 'UPSERTED' && (
+                                <Button
+                                    variant='contained'
+                                    sx={{
+                                        borderRadius: 2,
+                                        height: '100%',
+                                        backgroundImage: `linear-gradient(to right, #3f5efb, #fc466b)`,
+                                        '&:hover': {
+                                            backgroundImage: `linear-gradient(to right, #2b4efb, #fe2752)`
+                                        }
+                                    }}
+                                    startIcon={<IconZoomScan />}
+                                    onClick={() => showVectorStoreQuery(documentStore.id)}
+                                >
+                                    Retrieval Query
+                                </Button>
+                            )}
                         </ViewHeader>
                         {getSpecificDocumentStore.data?.whereUsed?.length > 0 && (
                             <Stack flexDirection='row' sx={{ gap: 2, alignItems: 'center', flexWrap: 'wrap' }}>
@@ -482,7 +575,13 @@ const DocumentStoreDetails = () => {
                                                             theme={theme}
                                                             onEditClick={() => openPreviewSettings(loader.id)}
                                                             onViewChunksClick={() => showStoredChunks(loader.id)}
-                                                            onDeleteClick={() => onLoaderDelete(loader)}
+                                                            onDeleteClick={() =>
+                                                                onLoaderDelete(
+                                                                    loader,
+                                                                    documentStore?.vectorStoreConfig,
+                                                                    documentStore?.recordManagerConfig
+                                                                )
+                                                            }
                                                         />
                                                     ))}
                                             </>
@@ -520,7 +619,15 @@ const DocumentStoreDetails = () => {
                     onDocLoaderSelected={onDocLoaderSelected}
                 />
             )}
-            <ConfirmDialog />
+            {showDeleteDocStoreDialog && (
+                <DeleteDocStoreDialog
+                    show={showDeleteDocStoreDialog}
+                    dialogProps={deleteDocStoreDialogProps}
+                    onCancel={() => setShowDeleteDocStoreDialog(false)}
+                    onDelete={onDocStoreDelete}
+                />
+            )}
+            {isBackdropLoading && <BackdropLoader open={isBackdropLoading} />}
         </>
     )
 }

packages/ui/src/views/docstore/DocumentStoreStatus.jsx

@@ -15,8 +15,10 @@ const DocumentStoreStatus = ({ status, isTableView }) => {
             case 'EMPTY':
                 return ['#673ab7', '#673ab7', '#673ab7']
             case 'SYNCING':
+            case 'UPSERTING':
                 return ['#fff8e1', '#ffe57f', '#ffc107']
             case 'SYNC':
+            case 'UPSERTED':
                 return ['#cdf5d8', '#00e676', '#00c853']
             case 'NEW':
                 return ['#e3f2fd', '#2196f3', '#1e88e5']

packages/ui/src/views/docstore/LoaderConfigPreviewChunks.jsx

@@ -279,7 +279,6 @@ const LoaderConfigPreviewChunks = () => {
     useEffect(() => {
         if (getNodeDetailsApi.data) {
             const nodeData = cloneDeep(initNode(getNodeDetailsApi.data, uuidv4()))
-
             // If this is a document store edit config, set the existing input values
             if (existingLoaderFromDocStoreTable && existingLoaderFromDocStoreTable.loaderConfig) {
                 nodeData.inputs = existingLoaderFromDocStoreTable.loaderConfig

packages/ui/src/views/docstore/UpsertHistoryDetailsDialog.jsx

@@ -0,0 +1,156 @@
+import { useState } from 'react'
+import { createPortal } from 'react-dom'
+import PropTypes from 'prop-types'
+import {
+    Box,
+    Paper,
+    Accordion,
+    AccordionSummary,
+    AccordionDetails,
+    Dialog,
+    DialogContent,
+    DialogTitle,
+    Typography,
+    Table,
+    TableBody,
+    TableContainer,
+    TableRow,
+    TableCell
+} from '@mui/material'
+import ExpandMoreIcon from '@mui/icons-material/ExpandMore'
+import { TableViewOnly } from '@/ui-component/table/Table'
+import StatsCard from '@/ui-component/cards/StatsCard'
+
+// const
+import { baseURL } from '@/store/constant'
+
+const UpsertHistoryDetailsDialog = ({ show, dialogProps, onCancel }) => {
+    const portalElement = document.getElementById('portal')
+    const [nodeConfigExpanded, setNodeConfigExpanded] = useState({})
+
+    const handleAccordionChange = (nodeLabel) => (event, isExpanded) => {
+        const accordianNodes = { ...nodeConfigExpanded }
+        accordianNodes[nodeLabel] = isExpanded
+        setNodeConfigExpanded(accordianNodes)
+    }
+    const component = show ? (
+        <Dialog
+            fullWidth
+            maxWidth='md'
+            open={show}
+            onClose={onCancel}
+            aria-labelledby='alert-dialog-title'
+            aria-describedby='alert-dialog-description'
+        >
+            <DialogTitle sx={{ fontSize: '1rem', p: 3, pb: 0 }} id='alert-dialog-title'>
+                {dialogProps.title}
+            </DialogTitle>
+            <DialogContent sx={{ display: 'flex', flexDirection: 'column', gap: 2, maxHeight: '75vh', position: 'relative', px: 3, pb: 3 }}>
+                <div
+                    style={{
+                        display: 'grid',
+                        gridTemplateColumns: 'repeat(4, minmax(0, 1fr))',
+                        gap: 5,
+                        marginTop: '10px'
+                    }}
+                >
+                    <StatsCard title='Added' stat={dialogProps.numAdded ?? 0} />
+                    <StatsCard title='Updated' stat={dialogProps.numUpdated ?? 0} />
+                    <StatsCard title='Skipped' stat={dialogProps.numSkipped ?? 0} />
+                    <StatsCard title='Deleted' stat={dialogProps.numDeleted ?? 0} />
+                </div>
+                <div>
+                    <TableContainer component={Paper}>
+                        <Table sx={{ minWidth: 650 }} aria-label='simple table'>
+                            <TableBody>
+                                <TableRow sx={{ '& td': { border: 0 } }}>
+                                    <TableCell sx={{ pb: 0, pt: 0 }} colSpan={6}>
+                                        <Box sx={{ mt: 1, mb: 2 }}>
+                                            {(dialogProps.flowData ?? []).map((node, index) => {
+                                                return (
+                                                    <Accordion
+                                                        expanded={nodeConfigExpanded[node.id] || false}
+                                                        onChange={handleAccordionChange(node.id)}
+                                                        key={index}
+                                                        disableGutters
+                                                    >
+                                                        <AccordionSummary
+                                                            expandIcon={<ExpandMoreIcon />}
+                                                            aria-controls={`nodes-accordian-${node.name}`}
+                                                            id={`nodes-accordian-header-${node.name}`}
+                                                        >
+                                                            <div style={{ display: 'flex', flexDirection: 'row', alignItems: 'center' }}>
+                                                                <div
+                                                                    style={{
+                                                                        width: 40,
+                                                                        height: 40,
+                                                                        marginRight: 10,
+                                                                        borderRadius: '50%',
+                                                                        backgroundColor: 'white'
+                                                                    }}
+                                                                >
+                                                                    <img
+                                                                        style={{
+                                                                            width: '100%',
+                                                                            height: '100%',
+                                                                            padding: 7,
+                                                                            borderRadius: '50%',
+                                                                            objectFit: 'contain'
+                                                                        }}
+                                                                        alt={node.name}
+                                                                        src={`${baseURL}/api/v1/node-icon/${node.name}`}
+                                                                    />
+                                                                </div>
+                                                                <Typography variant='h5'>{node.label}</Typography>
+                                                                <div
+                                                                    style={{
+                                                                        display: 'flex',
+                                                                        flexDirection: 'row',
+                                                                        width: 'max-content',
+                                                                        borderRadius: 15,
+                                                                        background: 'rgb(254,252,191)',
+                                                                        padding: 5,
+                                                                        paddingLeft: 10,
+                                                                        paddingRight: 10,
+                                                                        marginLeft: 10
+                                                                    }}
+                                                                >
+                                                                    <span style={{ color: 'rgb(116,66,16)', fontSize: '0.825rem' }}>
+                                                                        {node.id}
+                                                                    </span>
+                                                                </div>
+                                                            </div>
+                                                        </AccordionSummary>
+                                                        <AccordionDetails>
+                                                            {node.paramValues[0] && (
+                                                                <TableViewOnly
+                                                                    sx={{ minWidth: 150 }}
+                                                                    rows={node.paramValues}
+                                                                    columns={Object.keys(node.paramValues[0])}
+                                                                />
+                                                            )}
+                                                        </AccordionDetails>
+                                                    </Accordion>
+                                                )
+                                            })}
+                                        </Box>
+                                    </TableCell>
+                                </TableRow>
+                            </TableBody>
+                        </Table>
+                    </TableContainer>
+                </div>
+            </DialogContent>
+        </Dialog>
+    ) : null
+
+    return createPortal(component, portalElement)
+}
+
+UpsertHistoryDetailsDialog.propTypes = {
+    show: PropTypes.bool,
+    dialogProps: PropTypes.object,
+    onCancel: PropTypes.func
+}
+
+export default UpsertHistoryDetailsDialog

packages/ui/src/views/docstore/UpsertHistorySideDrawer.jsx

@@ -0,0 +1,113 @@
+import { useEffect, useState } from 'react'
+import PropTypes from 'prop-types'
+import moment from 'moment/moment'
+
+import { Stack, Button, Box, SwipeableDrawer } from '@mui/material'
+import { IconSquareRoundedChevronsRight } from '@tabler/icons-react'
+import {
+    Timeline,
+    TimelineConnector,
+    TimelineContent,
+    TimelineDot,
+    TimelineItem,
+    TimelineOppositeContent,
+    timelineOppositeContentClasses,
+    TimelineSeparator
+} from '@mui/lab'
+import HistoryEmptySVG from '@/assets/images/upsert_history_empty.svg'
+import vectorstoreApi from '@/api/vectorstore'
+import useApi from '@/hooks/useApi'
+
+const UpsertHistorySideDrawer = ({ show, dialogProps, onClickFunction, onSelectHistoryDetails }) => {
+    const onOpen = () => {}
+    const [upsertHistory, setUpsertHistory] = useState([])
+
+    const getUpsertHistoryApi = useApi(vectorstoreApi.getUpsertHistory)
+
+    useEffect(() => {
+        getUpsertHistoryApi.request(dialogProps.id)
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [dialogProps])
+
+    useEffect(() => {
+        if (getUpsertHistoryApi.data) {
+            setUpsertHistory(getUpsertHistoryApi.data)
+        }
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [getUpsertHistoryApi.data])
+
+    return (
+        <>
+            <SwipeableDrawer anchor='right' open={show} onClose={() => onClickFunction()} onOpen={onOpen}>
+                <Button startIcon={<IconSquareRoundedChevronsRight />} onClick={() => onClickFunction()}>
+                    Close
+                </Button>
+                <Box style={{ width: 350, margin: 10 }} role='presentation' onClick={onClickFunction}>
+                    <Timeline
+                        sx={{
+                            [`& .${timelineOppositeContentClasses.root}`]: {
+                                flex: 1
+                            }
+                        }}
+                    >
+                        {upsertHistory &&
+                            upsertHistory.map((history, index) => (
+                                <TimelineItem key={index}>
+                                    <TimelineOppositeContent>
+                                        {moment(history.date).format('DD-MMM-YYYY, hh:mm:ss A')}
+                                    </TimelineOppositeContent>
+                                    <TimelineSeparator style={{ marginTop: 5 }}>
+                                        <TimelineDot />
+                                        {index !== upsertHistory.length - 1 && <TimelineConnector />}
+                                    </TimelineSeparator>
+                                    <TimelineContent>
+                                        {history.result.numAdded !== undefined && history.result.numAdded > 0 && (
+                                            <Box sx={{ fontWeight: 500 }}>Added: {history.result.numAdded}</Box>
+                                        )}
+                                        {history.result.numUpdated !== undefined && history.result.numUpdated > 0 && (
+                                            <Box sx={{ fontWeight: 500 }}>Updated: {history.result.numUpdated}</Box>
+                                        )}
+                                        {history.result.numSkipped !== undefined && history.result.numSkipped > 0 && (
+                                            <Box sx={{ fontWeight: 500 }}>Skipped: {history.result.numSkipped}</Box>
+                                        )}
+                                        {history.result.numDeleted !== undefined && history.result.numDeleted > 0 && (
+                                            <Box sx={{ fontWeight: 500 }}>Deleted: {history.result.numDeleted}</Box>
+                                        )}
+                                        <Button
+                                            size='small'
+                                            sx={{ mt: 1, mb: 3, borderRadius: '25px' }}
+                                            variant='outlined'
+                                            onClick={() => onSelectHistoryDetails(history)}
+                                        >
+                                            Details
+                                        </Button>
+                                    </TimelineContent>
+                                </TimelineItem>
+                            ))}
+                        {upsertHistory.length === 0 && (
+                            <Stack sx={{ alignItems: 'center', justifyContent: 'center' }} flexDirection='column'>
+                                <Box sx={{ pb: 2, height: 'auto' }}>
+                                    <img
+                                        style={{ objectFit: 'cover', height: '10vh', width: 'auto' }}
+                                        src={HistoryEmptySVG}
+                                        alt='HistoryEmptySVG'
+                                    />
+                                </Box>
+                                <div>No Upsert History Yet</div>
+                            </Stack>
+                        )}
+                    </Timeline>
+                </Box>
+            </SwipeableDrawer>
+        </>
+    )
+}
+
+UpsertHistorySideDrawer.propTypes = {
+    show: PropTypes.bool,
+    dialogProps: PropTypes.object,
+    onClickFunction: PropTypes.func,
+    onSelectHistoryDetails: PropTypes.func
+}
+
+export default UpsertHistorySideDrawer

packages/ui/src/views/docstore/VectorStoreConfigure.jsx

@@ -0,0 +1,910 @@
+import { useEffect, useState } from 'react'
+import { useDispatch, useSelector } from 'react-redux'
+import { useNavigate } from 'react-router-dom'
+import { cloneDeep } from 'lodash'
+import { v4 as uuidv4 } from 'uuid'
+import moment from 'moment/moment'
+
+// material-ui
+import { Button, Stack, Grid, Box, Typography, IconButton, Stepper, Step, StepLabel } from '@mui/material'
+
+// project imports
+import MainCard from '@/ui-component/cards/MainCard'
+import ConfirmDialog from '@/ui-component/dialog/ConfirmDialog'
+import ComponentsListDialog from '@/views/docstore/ComponentsListDialog'
+import DocStoreInputHandler from '@/views/docstore/DocStoreInputHandler'
+import ViewHeader from '@/layout/MainLayout/ViewHeader'
+import { BackdropLoader } from '@/ui-component/loading/BackdropLoader'
+import ErrorBoundary from '@/ErrorBoundary'
+import UpsertResultDialog from '@/views/vectorstore/UpsertResultDialog'
+import UpsertHistorySideDrawer from './UpsertHistorySideDrawer'
+import UpsertHistoryDetailsDialog from './UpsertHistoryDetailsDialog'
+
+// API
+import documentsApi from '@/api/documentstore'
+import nodesApi from '@/api/nodes'
+
+// Hooks
+import useApi from '@/hooks/useApi'
+
+// Store
+import { closeSnackbar as closeSnackbarAction, enqueueSnackbar as enqueueSnackbarAction } from '@/store/actions'
+import { baseURL } from '@/store/constant'
+
+// icons
+import { IconX, IconEditCircle, IconRowInsertTop, IconDeviceFloppy, IconRefresh, IconClock } from '@tabler/icons-react'
+import Embeddings from '@mui/icons-material/DynamicFeed'
+import Storage from '@mui/icons-material/Storage'
+import DynamicFeed from '@mui/icons-material/Filter1'
+
+// utils
+import { initNode } from '@/utils/genericHelper'
+import useNotifier from '@/utils/useNotifier'
+
+// const
+const steps = ['Embeddings', 'Vector Store', 'Record Manager']
+
+const VectorStoreConfigure = () => {
+    const navigate = useNavigate()
+    const dispatch = useDispatch()
+    useNotifier()
+    const customization = useSelector((state) => state.customization)
+
+    const enqueueSnackbar = (...args) => dispatch(enqueueSnackbarAction(...args))
+    const closeSnackbar = (...args) => dispatch(closeSnackbarAction(...args))
+
+    const getSpecificDocumentStoreApi = useApi(documentsApi.getSpecificDocumentStore)
+    const insertIntoVectorStoreApi = useApi(documentsApi.insertIntoVectorStore)
+    const saveVectorStoreConfigApi = useApi(documentsApi.saveVectorStoreConfig)
+    const getEmbeddingNodeDetailsApi = useApi(nodesApi.getSpecificNode)
+    const getVectorStoreNodeDetailsApi = useApi(nodesApi.getSpecificNode)
+    const getRecordManagerNodeDetailsApi = useApi(nodesApi.getSpecificNode)
+
+    const [error, setError] = useState(null)
+    const [loading, setLoading] = useState(true)
+
+    const [documentStore, setDocumentStore] = useState({})
+    const [dialogProps, setDialogProps] = useState({})
+
+    const [showEmbeddingsListDialog, setShowEmbeddingsListDialog] = useState(false)
+    const [selectedEmbeddingsProvider, setSelectedEmbeddingsProvider] = useState({})
+
+    const [showVectorStoreListDialog, setShowVectorStoreListDialog] = useState(false)
+    const [selectedVectorStoreProvider, setSelectedVectorStoreProvider] = useState({})
+
+    const [showRecordManagerListDialog, setShowRecordManagerListDialog] = useState(false)
+    const [selectedRecordManagerProvider, setSelectedRecordManagerProvider] = useState({})
+    const [isRecordManagerUnavailable, setRecordManagerUnavailable] = useState(false)
+
+    const [showUpsertHistoryDialog, setShowUpsertHistoryDialog] = useState(false)
+    const [upsertResultDialogProps, setUpsertResultDialogProps] = useState({})
+
+    const [showUpsertHistorySideDrawer, setShowUpsertHistorySideDrawer] = useState(false)
+    const [upsertHistoryDrawerDialogProps, setUpsertHistoryDrawerDialogProps] = useState({})
+
+    const [showUpsertHistoryDetailsDialog, setShowUpsertHistoryDetailsDialog] = useState(false)
+    const [upsertDetailsDialogProps, setUpsertDetailsDialogProps] = useState({})
+
+    const onEmbeddingsSelected = (component) => {
+        const nodeData = cloneDeep(initNode(component, uuidv4()))
+        if (!showEmbeddingsListDialog && documentStore.embeddingConfig) {
+            nodeData.inputs = documentStore.embeddingConfig.config
+            nodeData.credential = documentStore.embeddingConfig.config.credential
+        }
+        setSelectedEmbeddingsProvider(nodeData)
+        setShowEmbeddingsListDialog(false)
+    }
+
+    const showEmbeddingsList = () => {
+        const dialogProp = {
+            title: 'Select Embeddings Provider'
+        }
+        setDialogProps(dialogProp)
+        setShowEmbeddingsListDialog(true)
+    }
+
+    const onVectorStoreSelected = (component) => {
+        const nodeData = cloneDeep(initNode(component, uuidv4()))
+        if (!nodeData.inputAnchors.find((anchor) => anchor.name === 'recordManager')) {
+            setRecordManagerUnavailable(true)
+            setSelectedRecordManagerProvider({})
+        } else {
+            setRecordManagerUnavailable(false)
+        }
+        if (!showVectorStoreListDialog && documentStore.vectorStoreConfig) {
+            nodeData.inputs = documentStore.vectorStoreConfig.config
+            nodeData.credential = documentStore.vectorStoreConfig.config.credential
+        }
+        setSelectedVectorStoreProvider(nodeData)
+        setShowVectorStoreListDialog(false)
+    }
+
+    const showVectorStoreList = () => {
+        const dialogProp = {
+            title: 'Select a Vector Store Provider'
+        }
+        setDialogProps(dialogProp)
+        setShowVectorStoreListDialog(true)
+    }
+
+    const onRecordManagerSelected = (component) => {
+        const nodeData = cloneDeep(initNode(component, uuidv4()))
+        if (!showRecordManagerListDialog && documentStore.recordManagerConfig) {
+            nodeData.inputs = documentStore.recordManagerConfig.config
+            nodeData.credential = documentStore.recordManagerConfig.config.credential
+        }
+        setSelectedRecordManagerProvider(nodeData)
+        setShowRecordManagerListDialog(false)
+    }
+
+    const showRecordManagerList = () => {
+        const dialogProp = {
+            title: 'Select a Record Manager'
+        }
+        setDialogProps(dialogProp)
+        setShowRecordManagerListDialog(true)
+    }
+
+    const showUpsertHistoryDrawer = () => {
+        const dialogProp = {
+            id: storeId
+        }
+        setUpsertHistoryDrawerDialogProps(dialogProp)
+        setShowUpsertHistorySideDrawer(true)
+    }
+
+    const onSelectHistoryDetails = (history) => {
+        const props = {
+            title: moment(history.date).format('DD-MMM-YYYY, hh:mm:ss A'),
+            numAdded: history.result.numAdded,
+            numUpdated: history.result.numUpdated,
+            numSkipped: history.result.numSkipped,
+            numDeleted: history.result.numDeleted,
+            flowData: history.flowData
+        }
+        setUpsertDetailsDialogProps(props)
+        setShowUpsertHistoryDetailsDialog(true)
+    }
+
+    const checkMandatoryFields = () => {
+        let canSubmit = true
+        const inputParams = (selectedVectorStoreProvider?.inputParams ?? []).filter((inputParam) => !inputParam.hidden)
+        for (const inputParam of inputParams) {
+            if (!inputParam.optional && (!selectedVectorStoreProvider.inputs[inputParam.name] || !selectedVectorStoreProvider.credential)) {
+                if (inputParam.type === 'credential' && !selectedVectorStoreProvider.credential) {
+                    canSubmit = false
+                    break
+                } else if (inputParam.type !== 'credential' && !selectedVectorStoreProvider.inputs[inputParam.name]) {
+                    canSubmit = false
+                    break
+                }
+            }
+        }
+
+        const inputParams2 = (selectedEmbeddingsProvider?.inputParams ?? []).filter((inputParam) => !inputParam.hidden)
+        for (const inputParam of inputParams2) {
+            if (!inputParam.optional && (!selectedEmbeddingsProvider.inputs[inputParam.name] || !selectedEmbeddingsProvider.credential)) {
+                if (inputParam.type === 'credential' && !selectedEmbeddingsProvider.credential) {
+                    canSubmit = false
+                    break
+                } else if (inputParam.type !== 'credential' && !selectedEmbeddingsProvider.inputs[inputParam.name]) {
+                    canSubmit = false
+                    break
+                }
+            }
+        }
+
+        if (!canSubmit) {
+            enqueueSnackbar({
+                message: 'Please fill in all mandatory fields.',
+                options: {
+                    key: new Date().getTime() + Math.random(),
+                    variant: 'warning',
+                    action: (key) => (
+                        <Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
+                            <IconX />
+                        </Button>
+                    )
+                }
+            })
+        }
+        return canSubmit
+    }
+
+    const prepareConfigData = () => {
+        const data = {
+            storeId: storeId
+        }
+        // Set embedding config
+        if (selectedEmbeddingsProvider.inputs) {
+            data.embeddingConfig = {}
+            data.embeddingName = selectedEmbeddingsProvider.name
+            Object.keys(selectedEmbeddingsProvider.inputs).map((key) => {
+                if (key === 'FLOWISE_CREDENTIAL_ID') {
+                    data.embeddingConfig['credential'] = selectedEmbeddingsProvider.inputs[key]
+                } else {
+                    data.embeddingConfig[key] = selectedEmbeddingsProvider.inputs[key]
+                }
+            })
+        } else {
+            data.embeddingConfig = null
+            data.embeddingName = ''
+        }
+
+        // Set vector store config
+        if (selectedVectorStoreProvider.inputs) {
+            data.vectorStoreConfig = {}
+            data.vectorStoreName = selectedVectorStoreProvider.name
+            Object.keys(selectedVectorStoreProvider.inputs).map((key) => {
+                if (key === 'FLOWISE_CREDENTIAL_ID') {
+                    data.vectorStoreConfig['credential'] = selectedVectorStoreProvider.inputs[key]
+                } else {
+                    data.vectorStoreConfig[key] = selectedVectorStoreProvider.inputs[key]
+                }
+            })
+        } else {
+            data.vectorStoreConfig = null
+            data.vectorStoreName = ''
+        }
+
+        // Set record manager config
+        if (selectedRecordManagerProvider.inputs) {
+            data.recordManagerConfig = {}
+            data.recordManagerName = selectedRecordManagerProvider.name
+            Object.keys(selectedRecordManagerProvider.inputs).map((key) => {
+                if (key === 'FLOWISE_CREDENTIAL_ID') {
+                    data.recordManagerConfig['credential'] = selectedRecordManagerProvider.inputs[key]
+                } else {
+                    data.recordManagerConfig[key] = selectedRecordManagerProvider.inputs[key]
+                }
+            })
+        } else {
+            data.recordManagerConfig = null
+            data.recordManagerName = ''
+        }
+
+        return data
+    }
+
+    const tryAndInsertIntoStore = () => {
+        if (checkMandatoryFields()) {
+            setLoading(true)
+            const data = prepareConfigData()
+            insertIntoVectorStoreApi.request(data)
+        }
+    }
+
+    const saveVectorStoreConfig = () => {
+        setLoading(true)
+        const data = prepareConfigData()
+        saveVectorStoreConfigApi.request(data)
+    }
+
+    const resetVectorStoreConfig = () => {
+        setSelectedEmbeddingsProvider({})
+        setSelectedVectorStoreProvider({})
+        setSelectedRecordManagerProvider({})
+    }
+
+    const getActiveStep = () => {
+        if (selectedRecordManagerProvider && Object.keys(selectedRecordManagerProvider).length > 0) {
+            return 3
+        }
+        if (selectedVectorStoreProvider && Object.keys(selectedVectorStoreProvider).length > 0) {
+            return 2
+        }
+        if (selectedEmbeddingsProvider && Object.keys(selectedEmbeddingsProvider).length > 0) {
+            return 1
+        }
+        return 0
+    }
+
+    const Steps = () => {
+        return (
+            <Box sx={{ width: '100%' }}>
+                <Stepper activeStep={getActiveStep()} alternativeLabel>
+                    {steps.map((label) => (
+                        <Step key={label}>
+                            <StepLabel>{label}</StepLabel>
+                        </Step>
+                    ))}
+                </Stepper>
+            </Box>
+        )
+    }
+
+    const isRecordManagerDisabled = () => {
+        return Object.keys(selectedVectorStoreProvider).length === 0 || isRecordManagerUnavailable
+    }
+
+    const isVectorStoreDisabled = () => {
+        return Object.keys(selectedEmbeddingsProvider).length === 0
+    }
+
+    useEffect(() => {
+        if (saveVectorStoreConfigApi.data) {
+            setLoading(false)
+            enqueueSnackbar({
+                message: 'Configuration saved successfully',
+                options: {
+                    key: new Date().getTime() + Math.random(),
+                    variant: 'success',
+                    action: (key) => (
+                        <Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
+                            <IconX />
+                        </Button>
+                    )
+                }
+            })
+        }
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [saveVectorStoreConfigApi.data])
+
+    useEffect(() => {
+        if (insertIntoVectorStoreApi.data) {
+            setLoading(false)
+            setShowUpsertHistoryDialog(true)
+            setUpsertResultDialogProps({ ...insertIntoVectorStoreApi.data, goToRetrievalQuery: true })
+        }
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [insertIntoVectorStoreApi.data])
+
+    useEffect(() => {
+        if (insertIntoVectorStoreApi.error) {
+            setLoading(false)
+            setError(insertIntoVectorStoreApi.error)
+        }
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [insertIntoVectorStoreApi.error])
+
+    useEffect(() => {
+        if (saveVectorStoreConfigApi.error) {
+            setLoading(false)
+            setError(saveVectorStoreConfigApi.error)
+        }
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [saveVectorStoreConfigApi.error])
+
+    const URLpath = document.location.pathname.toString().split('/')
+    const storeId = URLpath[URLpath.length - 1] === 'document-stores' ? '' : URLpath[URLpath.length - 1]
+    useEffect(() => {
+        getSpecificDocumentStoreApi.request(storeId)
+
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [])
+
+    useEffect(() => {
+        if (getSpecificDocumentStoreApi.data) {
+            const docStore = getSpecificDocumentStoreApi.data
+            setDocumentStore(docStore)
+            if (docStore.embeddingConfig) {
+                getEmbeddingNodeDetailsApi.request(docStore.embeddingConfig.name)
+            }
+            if (docStore.vectorStoreConfig) {
+                getVectorStoreNodeDetailsApi.request(docStore.vectorStoreConfig.name)
+            }
+            if (docStore.recordManagerConfig) {
+                getRecordManagerNodeDetailsApi.request(docStore.recordManagerConfig.name)
+            }
+            setLoading(false)
+        }
+
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [getSpecificDocumentStoreApi.data])
+
+    useEffect(() => {
+        if (getEmbeddingNodeDetailsApi.data) {
+            const node = getEmbeddingNodeDetailsApi.data
+            onEmbeddingsSelected(node)
+        }
+
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [getEmbeddingNodeDetailsApi.data])
+
+    useEffect(() => {
+        if (getVectorStoreNodeDetailsApi.data) {
+            const node = getVectorStoreNodeDetailsApi.data
+            onVectorStoreSelected(node)
+        }
+
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [getVectorStoreNodeDetailsApi.data])
+
+    useEffect(() => {
+        if (getRecordManagerNodeDetailsApi.data) {
+            const node = getRecordManagerNodeDetailsApi.data
+            onRecordManagerSelected(node)
+        }
+
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [getRecordManagerNodeDetailsApi.data])
+
+    useEffect(() => {
+        if (getSpecificDocumentStoreApi.error) {
+            setError(getSpecificDocumentStoreApi.error)
+        }
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [getSpecificDocumentStoreApi.error])
+
+    return (
+        <>
+            <MainCard>
+                {error ? (
+                    <ErrorBoundary error={error} />
+                ) : (
+                    <Stack flexDirection='column' sx={{ gap: 3 }}>
+                        <ViewHeader
+                            isBackButton={true}
+                            search={false}
+                            title={getSpecificDocumentStoreApi.data?.name}
+                            description='Configure Embeddings, Vector Store and Record Manager'
+                            onBack={() => navigate(-1)}
+                        >
+                            {(Object.keys(selectedEmbeddingsProvider).length > 0 ||
+                                Object.keys(selectedVectorStoreProvider).length > 0) && (
+                                <Button
+                                    variant='outlined'
+                                    color='error'
+                                    sx={{
+                                        borderRadius: 2,
+                                        height: '100%'
+                                    }}
+                                    startIcon={<IconRefresh />}
+                                    onClick={() => resetVectorStoreConfig()}
+                                >
+                                    Reset
+                                </Button>
+                            )}
+                            {(Object.keys(selectedEmbeddingsProvider).length > 0 ||
+                                Object.keys(selectedVectorStoreProvider).length > 0) && (
+                                <Button
+                                    variant='outlined'
+                                    color='secondary'
+                                    sx={{
+                                        borderRadius: 2,
+                                        height: '100%'
+                                    }}
+                                    startIcon={<IconDeviceFloppy />}
+                                    onClick={() => saveVectorStoreConfig()}
+                                >
+                                    Save Config
+                                </Button>
+                            )}
+                            {Object.keys(selectedEmbeddingsProvider).length > 0 && Object.keys(selectedVectorStoreProvider).length > 0 && (
+                                <Button
+                                    variant='contained'
+                                    sx={{
+                                        borderRadius: 2,
+                                        height: '100%',
+                                        backgroundImage: `linear-gradient(to right, #13547a, #2f9e91)`,
+                                        '&:hover': {
+                                            backgroundImage: `linear-gradient(to right, #0b3d5b, #1a8377)`
+                                        }
+                                    }}
+                                    startIcon={<IconRowInsertTop />}
+                                    onClick={() => tryAndInsertIntoStore()}
+                                >
+                                    Upsert
+                                </Button>
+                            )}
+                            <IconButton onClick={showUpsertHistoryDrawer} size='small' color='inherit' title='Upsert History'>
+                                <IconClock />
+                            </IconButton>
+                        </ViewHeader>
+                        <Steps />
+                        <Grid container spacing={1}>
+                            <Grid item xs={12} sm={4} md={4}>
+                                {Object.keys(selectedEmbeddingsProvider).length === 0 ? (
+                                    <Button
+                                        onClick={showEmbeddingsList}
+                                        fullWidth={true}
+                                        startIcon={<Embeddings style={{ background: 'transparent', height: 32, width: 32 }} />}
+                                        sx={{
+                                            color: customization?.isDarkMode ? 'white' : 'inherit',
+                                            borderRadius: '10px',
+                                            minHeight: '200px',
+                                            boxShadow: '0 2px 14px 0 rgb(32 40 45 / 20%)',
+                                            backgroundImage: customization?.isDarkMode
+                                                ? `linear-gradient(to right, #e654bc, #4b86e7)`
+                                                : `linear-gradient(to right, #fadef2, #cfdcf1)`,
+                                            '&:hover': {
+                                                backgroundImage: customization?.isDarkMode
+                                                    ? `linear-gradient(to right, #de32ac, #2d73e7)`
+                                                    : `linear-gradient(to right, #f6c2e7, #b4cbf1)`
+                                            }
+                                        }}
+                                    >
+                                        Select Embeddings
+                                    </Button>
+                                ) : (
+                                    <Box>
+                                        <Grid container spacing='2'>
+                                            <Grid item xs={12} md={12} lg={12} sm={12}>
+                                                <div
+                                                    style={{
+                                                        display: 'flex',
+                                                        flexDirection: 'column',
+                                                        paddingRight: 15
+                                                    }}
+                                                >
+                                                    <Box
+                                                        sx={{
+                                                            display: 'flex',
+                                                            alignItems: 'center',
+                                                            flexDirection: 'row',
+                                                            p: 1
+                                                        }}
+                                                    >
+                                                        <div
+                                                            style={{
+                                                                width: 40,
+                                                                height: 40,
+                                                                borderRadius: '50%',
+                                                                backgroundColor: 'white',
+                                                                display: 'flex',
+                                                                boxShadow: '0 2px 14px 0 rgb(32 40 45 / 25%)'
+                                                            }}
+                                                        >
+                                                            {selectedEmbeddingsProvider.label ? (
+                                                                <img
+                                                                    style={{
+                                                                        width: '100%',
+                                                                        height: '100%',
+                                                                        padding: 7,
+                                                                        borderRadius: '50%',
+                                                                        objectFit: 'contain'
+                                                                    }}
+                                                                    alt={selectedEmbeddingsProvider.label ?? 'embeddings'}
+                                                                    src={`${baseURL}/api/v1/node-icon/${selectedEmbeddingsProvider?.name}`}
+                                                                />
+                                                            ) : (
+                                                                <Embeddings color='black' />
+                                                            )}
+                                                        </div>
+                                                        <Typography sx={{ ml: 2 }} variant='h3'>
+                                                            {selectedEmbeddingsProvider.label}
+                                                        </Typography>
+                                                        <div style={{ flex: 1 }}></div>
+                                                        <div
+                                                            style={{
+                                                                display: 'flex',
+                                                                alignContent: 'center',
+                                                                flexDirection: 'row'
+                                                            }}
+                                                        >
+                                                            {Object.keys(selectedEmbeddingsProvider).length > 0 && (
+                                                                <>
+                                                                    <IconButton
+                                                                        variant='outlined'
+                                                                        sx={{ ml: 1 }}
+                                                                        color='secondary'
+                                                                        onClick={showEmbeddingsList}
+                                                                    >
+                                                                        <IconEditCircle />
+                                                                    </IconButton>
+                                                                </>
+                                                            )}
+                                                        </div>
+                                                    </Box>
+                                                    {selectedEmbeddingsProvider &&
+                                                        Object.keys(selectedEmbeddingsProvider).length > 0 &&
+                                                        (selectedEmbeddingsProvider.inputParams ?? [])
+                                                            .filter((inputParam) => !inputParam.hidden)
+                                                            .map((inputParam, index) => (
+                                                                <DocStoreInputHandler
+                                                                    key={index}
+                                                                    data={selectedEmbeddingsProvider}
+                                                                    inputParam={inputParam}
+                                                                    isAdditionalParams={inputParam.additionalParams}
+                                                                />
+                                                            ))}
+                                                </div>
+                                            </Grid>
+                                        </Grid>
+                                    </Box>
+                                )}
+                            </Grid>
+                            <Grid item xs={12} sm={4} md={4}>
+                                {Object.keys(selectedVectorStoreProvider).length === 0 ? (
+                                    <Button
+                                        onClick={showVectorStoreList}
+                                        fullWidth={true}
+                                        startIcon={<Storage style={{ background: 'transparent', height: 32, width: 32 }} />}
+                                        sx={{
+                                            color: customization?.isDarkMode ? 'white' : 'inherit',
+                                            borderRadius: '10px',
+                                            minHeight: '200px',
+                                            opacity: isVectorStoreDisabled() ? 0.7 : 1,
+                                            boxShadow: isVectorStoreDisabled() ? 'none' : '0 2px 14px 0 rgb(32 40 45 / 20%)',
+                                            backgroundImage: customization?.isDarkMode
+                                                ? `linear-gradient(to right, #4d8ef1, #f1de5c)`
+                                                : `linear-gradient(to right, #b9d0f4, #fef9d7)`,
+                                            '&:hover': {
+                                                backgroundImage: customization?.isDarkMode
+                                                    ? `linear-gradient(to right, #2576f2, #f0d72e)`
+                                                    : `linear-gradient(to right, #9cbdf2, #fcf3b6)`
+                                            }
+                                        }}
+                                        disabled={isVectorStoreDisabled()}
+                                    >
+                                        Select Vector Store
+                                    </Button>
+                                ) : (
+                                    <Box>
+                                        <Grid container spacing='2'>
+                                            <Grid item xs={12} md={12} lg={12} sm={12}>
+                                                <div
+                                                    style={{
+                                                        display: 'flex',
+                                                        flexDirection: 'column',
+                                                        paddingRight: 15
+                                                    }}
+                                                >
+                                                    <Box
+                                                        sx={{
+                                                            display: 'flex',
+                                                            alignItems: 'center',
+                                                            flexDirection: 'row',
+                                                            p: 1
+                                                        }}
+                                                    >
+                                                        <div
+                                                            style={{
+                                                                width: 40,
+                                                                height: 40,
+                                                                borderRadius: '50%',
+                                                                backgroundColor: 'white',
+                                                                display: 'flex',
+                                                                alignItems: 'center',
+                                                                justifyContent: 'center',
+                                                                boxShadow: '0 2px 14px 0 rgb(32 40 45 / 25%)'
+                                                            }}
+                                                        >
+                                                            {selectedVectorStoreProvider.label ? (
+                                                                <img
+                                                                    style={{
+                                                                        width: '100%',
+                                                                        height: '100%',
+                                                                        padding: 7,
+                                                                        borderRadius: '50%',
+                                                                        objectFit: 'contain'
+                                                                    }}
+                                                                    alt={selectedVectorStoreProvider.label ?? 'embeddings'}
+                                                                    src={`${baseURL}/api/v1/node-icon/${selectedVectorStoreProvider?.name}`}
+                                                                />
+                                                            ) : (
+                                                                <Embeddings color='black' />
+                                                            )}
+                                                        </div>
+                                                        <Typography sx={{ ml: 2 }} variant='h3'>
+                                                            {selectedVectorStoreProvider.label}
+                                                        </Typography>
+                                                        <div style={{ flex: 1 }}></div>
+                                                        <div
+                                                            style={{
+                                                                display: 'flex',
+                                                                alignContent: 'center',
+                                                                flexDirection: 'row'
+                                                            }}
+                                                        >
+                                                            {Object.keys(selectedVectorStoreProvider).length > 0 && (
+                                                                <>
+                                                                    <IconButton
+                                                                        variant='outlined'
+                                                                        sx={{ ml: 1 }}
+                                                                        color='secondary'
+                                                                        onClick={showVectorStoreList}
+                                                                    >
+                                                                        <IconEditCircle />
+                                                                    </IconButton>
+                                                                </>
+                                                            )}
+                                                        </div>
+                                                    </Box>
+                                                    {selectedVectorStoreProvider &&
+                                                        Object.keys(selectedVectorStoreProvider).length > 0 &&
+                                                        (selectedVectorStoreProvider.inputParams ?? [])
+                                                            .filter((inputParam) => !inputParam.hidden)
+                                                            .map((inputParam, index) => (
+                                                                <DocStoreInputHandler
+                                                                    key={index}
+                                                                    data={selectedVectorStoreProvider}
+                                                                    inputParam={inputParam}
+                                                                    isAdditionalParams={inputParam.additionalParams}
+                                                                />
+                                                            ))}
+                                                </div>
+                                            </Grid>
+                                        </Grid>
+                                    </Box>
+                                )}
+                            </Grid>
+                            <Grid item xs={12} sm={4} md={4}>
+                                {Object.keys(selectedRecordManagerProvider).length === 0 ? (
+                                    <Button
+                                        onClick={showRecordManagerList}
+                                        fullWidth={true}
+                                        startIcon={
+                                            isRecordManagerUnavailable ? (
+                                                <></>
+                                            ) : (
+                                                <DynamicFeed style={{ background: 'transparent', height: 32, width: 32 }} />
+                                            )
+                                        }
+                                        sx={{
+                                            color: customization?.isDarkMode ? 'white' : 'inherit',
+                                            borderRadius: '10px',
+                                            minHeight: '200px',
+                                            opacity: isRecordManagerDisabled() ? 0.7 : 1,
+                                            boxShadow: isRecordManagerDisabled() ? 'none' : '0 2px 14px 0 rgb(32 40 45 / 20%)',
+                                            backgroundImage: customization?.isDarkMode
+                                                ? `linear-gradient(to right, #f5db3f, #42daa7)`
+                                                : `linear-gradient(to right, #f9f1c0, #c7f1e3)`,
+                                            '&:hover': {
+                                                backgroundImage: customization?.isDarkMode
+                                                    ? `linear-gradient(to right, #d9c238, #3dc295)`
+                                                    : `linear-gradient(to right, #f6e99b, #a0f2d7)`
+                                            }
+                                        }}
+                                        disabled={isRecordManagerDisabled()}
+                                    >
+                                        {isRecordManagerUnavailable
+                                            ? 'Record Manager is not applicable for selected Vector Store'
+                                            : 'Select Record Manager'}
+                                    </Button>
+                                ) : (
+                                    <Box>
+                                        <Grid container spacing='2'>
+                                            <Grid item xs={12} md={12} lg={12} sm={12}>
+                                                <div
+                                                    style={{
+                                                        display: 'flex',
+                                                        flexDirection: 'column',
+                                                        paddingRight: 15
+                                                    }}
+                                                >
+                                                    <Box
+                                                        sx={{
+                                                            display: 'flex',
+                                                            alignItems: 'center',
+                                                            flexDirection: 'row',
+                                                            p: 1
+                                                        }}
+                                                    >
+                                                        <div
+                                                            style={{
+                                                                width: 40,
+                                                                height: 40,
+                                                                borderRadius: '50%',
+                                                                backgroundColor: 'white',
+                                                                display: 'flex',
+                                                                alignItems: 'center',
+                                                                justifyContent: 'center',
+                                                                boxShadow: '0 2px 14px 0 rgb(32 40 45 / 25%)'
+                                                            }}
+                                                        >
+                                                            {selectedRecordManagerProvider.label ? (
+                                                                <img
+                                                                    style={{
+                                                                        width: '100%',
+                                                                        height: '100%',
+                                                                        padding: 7,
+                                                                        borderRadius: '50%',
+                                                                        objectFit: 'contain'
+                                                                    }}
+                                                                    alt={selectedRecordManagerProvider.label ?? 'embeddings'}
+                                                                    src={`${baseURL}/api/v1/node-icon/${selectedRecordManagerProvider?.name}`}
+                                                                />
+                                                            ) : (
+                                                                <Embeddings color='black' />
+                                                            )}
+                                                        </div>
+                                                        <Typography sx={{ ml: 2 }} variant='h3'>
+                                                            {selectedRecordManagerProvider.label}
+                                                        </Typography>
+                                                        <div style={{ flex: 1 }}></div>
+                                                        <div
+                                                            style={{
+                                                                display: 'flex',
+                                                                alignContent: 'center',
+                                                                flexDirection: 'row'
+                                                            }}
+                                                        >
+                                                            {Object.keys(selectedRecordManagerProvider).length > 0 && (
+                                                                <>
+                                                                    <IconButton
+                                                                        variant='outlined'
+                                                                        sx={{ ml: 1 }}
+                                                                        color='secondary'
+                                                                        onClick={showRecordManagerList}
+                                                                    >
+                                                                        <IconEditCircle />
+                                                                    </IconButton>
+                                                                </>
+                                                            )}
+                                                        </div>
+                                                    </Box>
+                                                    {selectedRecordManagerProvider &&
+                                                        Object.keys(selectedRecordManagerProvider).length > 0 &&
+                                                        (selectedRecordManagerProvider.inputParams ?? [])
+                                                            .filter((inputParam) => !inputParam.hidden)
+                                                            .map((inputParam, index) => (
+                                                                <>
+                                                                    <DocStoreInputHandler
+                                                                        key={index}
+                                                                        data={selectedRecordManagerProvider}
+                                                                        inputParam={inputParam}
+                                                                        isAdditionalParams={inputParam.additionalParams}
+                                                                    />
+                                                                </>
+                                                            ))}
+                                                </div>
+                                            </Grid>
+                                        </Grid>
+                                    </Box>
+                                )}
+                            </Grid>
+                        </Grid>
+                    </Stack>
+                )}
+            </MainCard>
+
+            {showEmbeddingsListDialog && (
+                <ComponentsListDialog
+                    show={showEmbeddingsListDialog}
+                    dialogProps={dialogProps}
+                    onCancel={() => setShowEmbeddingsListDialog(false)}
+                    apiCall={documentsApi.getEmbeddingProviders}
+                    onSelected={onEmbeddingsSelected}
+                />
+            )}
+            {showVectorStoreListDialog && (
+                <ComponentsListDialog
+                    show={showVectorStoreListDialog}
+                    dialogProps={dialogProps}
+                    onCancel={() => setShowVectorStoreListDialog(false)}
+                    apiCall={documentsApi.getVectorStoreProviders}
+                    onSelected={onVectorStoreSelected}
+                />
+            )}
+            {showRecordManagerListDialog && (
+                <ComponentsListDialog
+                    show={showRecordManagerListDialog}
+                    dialogProps={dialogProps}
+                    onCancel={() => setShowRecordManagerListDialog(false)}
+                    apiCall={documentsApi.getRecordManagerProviders}
+                    onSelected={onRecordManagerSelected}
+                />
+            )}
+            {showUpsertHistoryDialog && (
+                <UpsertResultDialog
+                    show={showUpsertHistoryDialog}
+                    dialogProps={upsertResultDialogProps}
+                    onCancel={() => {
+                        setShowUpsertHistoryDialog(false)
+                    }}
+                    onGoToRetrievalQuery={() => navigate('/document-stores/query/' + storeId)}
+                ></UpsertResultDialog>
+            )}
+            {showUpsertHistorySideDrawer && (
+                <UpsertHistorySideDrawer
+                    show={showUpsertHistorySideDrawer}
+                    dialogProps={upsertHistoryDrawerDialogProps}
+                    onClickFunction={() => setShowUpsertHistorySideDrawer(false)}
+                    onSelectHistoryDetails={onSelectHistoryDetails}
+                />
+            )}
+            {showUpsertHistoryDetailsDialog && (
+                <UpsertHistoryDetailsDialog
+                    show={showUpsertHistoryDetailsDialog}
+                    dialogProps={upsertDetailsDialogProps}
+                    onCancel={() => setShowUpsertHistoryDetailsDialog(false)}
+                />
+            )}
+            <ConfirmDialog />
+            {loading && <BackdropLoader open={loading} />}
+        </>
+    )
+}
+
+export default VectorStoreConfigure

packages/ui/src/views/docstore/VectorStoreQuery.jsx

@@ -0,0 +1,483 @@
+import { useEffect, useState, useRef } from 'react'
+import { useDispatch, useSelector } from 'react-redux'
+import { useNavigate } from 'react-router-dom'
+import ReactJson from 'flowise-react-json-view'
+import { cloneDeep } from 'lodash'
+import { v4 as uuidv4 } from 'uuid'
+
+// material-ui
+import { Box, Card, Grid, Stack, Typography, OutlinedInput, IconButton, Button } from '@mui/material'
+import Embeddings from '@mui/icons-material/DynamicFeed'
+import { useTheme, styled } from '@mui/material/styles'
+import CardContent from '@mui/material/CardContent'
+import chunks_emptySVG from '@/assets/images/chunks_empty.svg'
+import { IconSearch, IconFileStack, IconDeviceFloppy, IconX } from '@tabler/icons-react'
+
+// project imports
+import MainCard from '@/ui-component/cards/MainCard'
+import { BackdropLoader } from '@/ui-component/loading/BackdropLoader'
+import ConfirmDialog from '@/ui-component/dialog/ConfirmDialog'
+import ExpandedChunkDialog from './ExpandedChunkDialog'
+import ViewHeader from '@/layout/MainLayout/ViewHeader'
+import DocStoreInputHandler from '@/views/docstore/DocStoreInputHandler'
+
+// API
+import documentsApi from '@/api/documentstore'
+import nodesApi from '@/api/nodes'
+
+// Hooks
+import useApi from '@/hooks/useApi'
+import useNotifier from '@/utils/useNotifier'
+import { baseURL } from '@/store/constant'
+import { initNode } from '@/utils/genericHelper'
+import { closeSnackbar as closeSnackbarAction, enqueueSnackbar as enqueueSnackbarAction } from '@/store/actions'
+
+const CardWrapper = styled(MainCard)(({ theme }) => ({
+    background: theme.palette.card.main,
+    color: theme.darkTextPrimary,
+    overflow: 'auto',
+    position: 'relative',
+    boxShadow: '0 2px 14px 0 rgb(32 40 45 / 8%)',
+    cursor: 'pointer',
+    '&:hover': {
+        background: theme.palette.card.hover,
+        boxShadow: '0 2px 14px 0 rgb(32 40 45 / 20%)'
+    },
+    maxHeight: '250px',
+    minHeight: '250px',
+    maxWidth: '100%',
+    overflowWrap: 'break-word',
+    whiteSpace: 'pre-line',
+    padding: 1
+}))
+
+const VectorStoreQuery = () => {
+    const customization = useSelector((state) => state.customization)
+    const navigate = useNavigate()
+    const theme = useTheme()
+    const dispatch = useDispatch()
+    const inputRef = useRef(null)
+
+    useNotifier()
+
+    const enqueueSnackbar = (...args) => dispatch(enqueueSnackbarAction(...args))
+    const closeSnackbar = (...args) => dispatch(closeSnackbarAction(...args))
+
+    const URLpath = document.location.pathname.toString().split('/')
+    const storeId = URLpath[URLpath.length - 1] === 'document-stores' ? '' : URLpath[URLpath.length - 1]
+
+    const [documentChunks, setDocumentChunks] = useState([])
+    const [loading, setLoading] = useState(false)
+    const [showExpandedChunkDialog, setShowExpandedChunkDialog] = useState(false)
+    const [expandedChunkDialogProps, setExpandedChunkDialogProps] = useState({})
+    const [documentStore, setDocumentStore] = useState({})
+    const [query, setQuery] = useState('')
+
+    const [timeTaken, setTimeTaken] = useState(-1)
+    const [retrievalError, setRetrievalError] = useState(undefined)
+
+    const getSpecificDocumentStoreApi = useApi(documentsApi.getSpecificDocumentStore)
+    const queryVectorStoreApi = useApi(documentsApi.queryVectorStore)
+
+    const getVectorStoreNodeDetailsApi = useApi(nodesApi.getSpecificNode)
+    const [selectedVectorStoreProvider, setSelectedVectorStoreProvider] = useState({})
+
+    const chunkSelected = (chunkId, selectedChunkNumber) => {
+        const selectedChunk = documentChunks.find((chunk) => chunk.id === chunkId)
+        const dialogProps = {
+            data: {
+                selectedChunk,
+                selectedChunkNumber
+            }
+        }
+        setExpandedChunkDialogProps(dialogProps)
+        setShowExpandedChunkDialog(true)
+    }
+
+    const handleEnter = (e) => {
+        // Check if IME composition is in progress
+        const isIMEComposition = e.isComposing || e.keyCode === 229
+        if (e.key === 'Enter' && query && !isIMEComposition) {
+            if (!e.shiftKey && query) {
+                if (inputRef.current) {
+                    inputRef.current.blur()
+                }
+                doQuery()
+            }
+        } else if (e.key === 'Enter') {
+            e.preventDefault()
+        }
+    }
+
+    const doQuery = () => {
+        setLoading(true)
+        const data = {
+            query: query,
+            storeId: storeId,
+            inputs: selectedVectorStoreProvider.inputs
+        }
+        queryVectorStoreApi.request(data)
+    }
+
+    const saveConfig = async () => {
+        setLoading(true)
+        const data = {
+            storeId: storeId
+        }
+
+        if (selectedVectorStoreProvider.inputs) {
+            data.vectorStoreConfig = {}
+            data.vectorStoreName = selectedVectorStoreProvider.name
+            Object.keys(selectedVectorStoreProvider.inputs).map((key) => {
+                if (key === 'FLOWISE_CREDENTIAL_ID') {
+                    data.vectorStoreConfig['credential'] = selectedVectorStoreProvider.inputs[key]
+                } else {
+                    data.vectorStoreConfig[key] = selectedVectorStoreProvider.inputs[key]
+                }
+            })
+        }
+
+        try {
+            const updateResp = await documentsApi.updateVectorStoreConfig(data)
+            setLoading(false)
+            if (updateResp.data) {
+                enqueueSnackbar({
+                    message: 'Vector Store Config Successfully Updated',
+                    options: {
+                        key: new Date().getTime() + Math.random(),
+                        variant: 'success',
+                        action: (key) => (
+                            <Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
+                                <IconX />
+                            </Button>
+                        )
+                    }
+                })
+            }
+        } catch (error) {
+            setLoading(false)
+            const errorData = error.response?.data || `${error.response?.status}: ${error.response?.statusText}`
+            enqueueSnackbar({
+                message: `Failed to update vector store config: ${errorData}`,
+                options: {
+                    key: new Date().getTime() + Math.random(),
+                    variant: 'error',
+                    persist: true,
+                    action: (key) => (
+                        <Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
+                            <IconX />
+                        </Button>
+                    )
+                }
+            })
+        }
+    }
+
+    useEffect(() => {
+        if (queryVectorStoreApi.data) {
+            setDocumentChunks(queryVectorStoreApi.data.docs)
+            setTimeTaken(queryVectorStoreApi.data.timeTaken)
+            setRetrievalError(undefined)
+            setLoading(false)
+            if (inputRef.current) {
+                inputRef.current.focus()
+            }
+        }
+
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [queryVectorStoreApi.data])
+
+    useEffect(() => {
+        if (queryVectorStoreApi.error) {
+            if (queryVectorStoreApi.error.response?.data?.message) {
+                const message = queryVectorStoreApi.error.response.data.message
+                // remove the text 'documentStoreServices.queryVectorStore - ' from the error message to make it readable
+                setRetrievalError(message.replace('documentStoreServices.queryVectorStore - ', ''))
+                setDocumentChunks([])
+                setTimeTaken(-1)
+            }
+            setLoading(false)
+        }
+
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [queryVectorStoreApi.error])
+
+    useEffect(() => {
+        if (getVectorStoreNodeDetailsApi.data) {
+            const node = getVectorStoreNodeDetailsApi.data
+            fetchVectorStoreDetails(node)
+        }
+
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [getVectorStoreNodeDetailsApi.data])
+
+    const fetchVectorStoreDetails = (component) => {
+        const nodeData = cloneDeep(initNode(component, uuidv4()))
+        if (documentStore.vectorStoreConfig) {
+            nodeData.inputs = documentStore.vectorStoreConfig.config
+            nodeData.credential = documentStore.vectorStoreConfig.config.credential
+        }
+        setSelectedVectorStoreProvider(nodeData)
+    }
+
+    useEffect(() => {
+        getSpecificDocumentStoreApi.request(storeId)
+
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [])
+
+    useEffect(() => {
+        if (getSpecificDocumentStoreApi.data) {
+            setDocumentStore(getSpecificDocumentStoreApi.data)
+            const vectorStoreConfig = getSpecificDocumentStoreApi.data.vectorStoreConfig
+            if (vectorStoreConfig) {
+                getVectorStoreNodeDetailsApi.request(vectorStoreConfig.name)
+            }
+            setLoading(false)
+        }
+
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [getSpecificDocumentStoreApi.data])
+
+    return (
+        <>
+            <MainCard style={{ position: 'relative' }}>
+                <Stack flexDirection='column' sx={{ gap: 1 }}>
+                    <ViewHeader
+                        isBackButton={true}
+                        search={false}
+                        title={documentStore?.name || 'Document Store'}
+                        description='Retrieval Playground - Test your vector store retrieval settings'
+                        onBack={() => navigate(-1)}
+                    >
+                        <Button
+                            variant='outlined'
+                            color='secondary'
+                            sx={{ borderRadius: 2, height: '100%' }}
+                            startIcon={<IconDeviceFloppy />}
+                            onClick={saveConfig}
+                        >
+                            Save Config
+                        </Button>
+                    </ViewHeader>
+                    <div style={{ width: '100%' }}></div>
+                    <div>
+                        <Grid container spacing={2}>
+                            <Grid sx={{ ml: 1, mr: 1 }} item xs={12} sm={12} md={12} lg={12}>
+                                <Box>
+                                    <div style={{ display: 'flex', flexDirection: 'row' }}>
+                                        <Typography variant='overline'>
+                                            Enter your Query<span style={{ color: 'red' }}>&nbsp;*</span>
+                                        </Typography>
+
+                                        <div style={{ flexGrow: 1 }}></div>
+                                    </div>
+                                    <OutlinedInput
+                                        size='small'
+                                        multiline={true}
+                                        rows={4}
+                                        sx={{ mt: 1 }}
+                                        type='string'
+                                        fullWidth
+                                        inputRef={inputRef}
+                                        onChange={(e) => setQuery(e.target.value)}
+                                        onKeyDown={handleEnter}
+                                        value={query ?? ''}
+                                        endAdornment={
+                                            <IconButton variant='contained' onClick={doQuery}>
+                                                <IconSearch />
+                                            </IconButton>
+                                        }
+                                    />
+                                </Box>
+                            </Grid>
+                            <Grid sx={{ ml: 1, mr: 1, mt: 1 }} container spacing={1}>
+                                <Grid item xs={12} sm={4} md={4}>
+                                    <Box>
+                                        <Grid container spacing='2'>
+                                            <Grid item xs={12} md={12} lg={12} sm={12}>
+                                                <div
+                                                    style={{
+                                                        display: 'flex',
+                                                        flexDirection: 'column',
+                                                        paddingRight: 15,
+                                                        paddingTop: 5
+                                                    }}
+                                                >
+                                                    <Box
+                                                        sx={{
+                                                            display: 'flex',
+                                                            alignItems: 'center',
+                                                            flexDirection: 'row',
+                                                            p: 1
+                                                        }}
+                                                    >
+                                                        <div
+                                                            style={{
+                                                                width: 40,
+                                                                height: 40,
+                                                                borderRadius: '50%',
+                                                                backgroundColor: 'white',
+                                                                display: 'flex',
+                                                                alignItems: 'center',
+                                                                justifyContent: 'center',
+                                                                boxShadow: '0 2px 14px 0 rgb(32 40 45 / 25%)'
+                                                            }}
+                                                        >
+                                                            {selectedVectorStoreProvider.label ? (
+                                                                <img
+                                                                    style={{
+                                                                        width: '100%',
+                                                                        height: '100%',
+                                                                        padding: 7,
+                                                                        borderRadius: '50%',
+                                                                        objectFit: 'contain'
+                                                                    }}
+                                                                    alt={selectedVectorStoreProvider.label ?? 'embeddings'}
+                                                                    src={`${baseURL}/api/v1/node-icon/${selectedVectorStoreProvider?.name}`}
+                                                                />
+                                                            ) : (
+                                                                <Embeddings color='black' />
+                                                            )}
+                                                        </div>
+                                                        <Typography sx={{ ml: 2 }} variant='h3'>
+                                                            {selectedVectorStoreProvider.label}
+                                                        </Typography>
+                                                        <div style={{ flex: 1 }}></div>
+                                                    </Box>
+                                                    {selectedVectorStoreProvider &&
+                                                        Object.keys(selectedVectorStoreProvider).length > 0 &&
+                                                        (selectedVectorStoreProvider.inputParams ?? [])
+                                                            .filter((inputParam) => !inputParam.hidden)
+                                                            .map((inputParam, index) => (
+                                                                <DocStoreInputHandler
+                                                                    key={index}
+                                                                    data={selectedVectorStoreProvider}
+                                                                    inputParam={inputParam}
+                                                                    isAdditionalParams={inputParam.additionalParams}
+                                                                />
+                                                            ))}
+                                                </div>
+                                            </Grid>
+                                        </Grid>
+                                    </Box>
+                                </Grid>
+                                <Grid item xs={12} sm={8} md={8}>
+                                    <Box
+                                        sx={{
+                                            display: 'flex',
+                                            alignItems: 'center',
+                                            flexDirection: 'row',
+                                            p: 1,
+                                            paddingTop: 2,
+                                            marginBottom: 4
+                                        }}
+                                    >
+                                        <div
+                                            style={{
+                                                width: 40,
+                                                height: 40,
+                                                borderRadius: '50%',
+                                                backgroundColor: 'white',
+                                                display: 'flex',
+                                                alignItems: 'center',
+                                                justifyContent: 'center',
+                                                boxShadow: '0 2px 14px 0 rgb(32 40 45 / 25%)'
+                                            }}
+                                        >
+                                            <IconFileStack
+                                                style={{
+                                                    width: '100%',
+                                                    height: '100%',
+                                                    padding: 7,
+                                                    borderRadius: '50%',
+                                                    objectFit: 'contain'
+                                                }}
+                                            />
+                                        </div>
+                                        <Typography sx={{ ml: 2 }} variant='h3'>
+                                            Retrieved Documents
+                                            {timeTaken > -1 && (
+                                                <Typography variant='body2' sx={{ color: 'gray' }}>
+                                                    Count: {documentChunks.length}. Time taken: {timeTaken} millis.
+                                                </Typography>
+                                            )}
+                                            {retrievalError && (
+                                                <Typography variant='body2' sx={{ color: 'gray' }}>
+                                                    {retrievalError}
+                                                </Typography>
+                                            )}
+                                        </Typography>
+                                        <div style={{ flex: 1 }}></div>
+                                    </Box>
+                                    {!documentChunks.length && (
+                                        <div
+                                            style={{
+                                                display: 'flex',
+                                                flexDirection: 'column',
+                                                alignItems: 'center',
+                                                width: '100%'
+                                            }}
+                                        >
+                                            <Box sx={{ mt: 5, p: 2, height: 'auto' }}>
+                                                <img
+                                                    style={{ objectFit: 'cover', height: '16vh', width: 'auto' }}
+                                                    src={chunks_emptySVG}
+                                                    alt='chunks_emptySVG'
+                                                />
+                                            </Box>
+                                            <div>No Documents Retrieved</div>
+                                        </div>
+                                    )}
+                                    <Grid container spacing={2}>
+                                        {documentChunks?.length > 0 &&
+                                            documentChunks.map((row, index) => (
+                                                <Grid item lg={6} md={6} sm={6} xs={6} key={index}>
+                                                    <CardWrapper
+                                                        content={false}
+                                                        onClick={() => chunkSelected(row.id, row.chunkNo)}
+                                                        sx={{ border: 1, borderColor: theme.palette.grey[900] + 25, borderRadius: 2 }}
+                                                    >
+                                                        <Card>
+                                                            <CardContent sx={{ p: 2 }}>
+                                                                <Typography sx={{ wordWrap: 'break-word', mb: 1 }} variant='h5'>
+                                                                    {`#${row.chunkNo}. Characters: ${row.pageContent.length}`}
+                                                                </Typography>
+                                                                <Typography sx={{ wordWrap: 'break-word' }} variant='body2'>
+                                                                    {row.pageContent}
+                                                                </Typography>
+                                                                <ReactJson
+                                                                    theme={customization.isDarkMode ? 'ocean' : 'rjv-default'}
+                                                                    style={{ paddingTop: 10 }}
+                                                                    src={row.metadata || {}}
+                                                                    name={null}
+                                                                    quotesOnKeys={false}
+                                                                    enableClipboard={false}
+                                                                    displayDataTypes={false}
+                                                                    collapsed={true}
+                                                                />
+                                                            </CardContent>
+                                                        </Card>
+                                                    </CardWrapper>
+                                                </Grid>
+                                            ))}
+                                    </Grid>
+                                </Grid>
+                            </Grid>
+                        </Grid>
+                    </div>
+                </Stack>
+            </MainCard>
+            <ConfirmDialog />
+            <ExpandedChunkDialog
+                show={showExpandedChunkDialog}
+                dialogProps={expandedChunkDialogProps}
+                onCancel={() => setShowExpandedChunkDialog(false)}
+                isReadOnly={true}
+            ></ExpandedChunkDialog>
+            {loading && <BackdropLoader open={loading} />}
+        </>
+    )
+}
+
+export default VectorStoreQuery

packages/ui/src/views/vectorstore/UpsertResultDialog.jsx

@@ -6,8 +6,9 @@ import ReactJson from 'flowise-react-json-view'
 import { Typography, Card, CardContent, Button, Dialog, DialogActions, DialogContent, DialogTitle } from '@mui/material'
 import StatsCard from '@/ui-component/cards/StatsCard'
 import { HIDE_CANVAS_DIALOG, SHOW_CANVAS_DIALOG } from '@/store/actions'
+import { IconZoomScan } from '@tabler/icons-react'
 
-const UpsertResultDialog = ({ show, dialogProps, onCancel }) => {
+const UpsertResultDialog = ({ show, dialogProps, onCancel, onGoToRetrievalQuery }) => {
     const portalElement = document.getElementById('portal')
     const dispatch = useDispatch()
     const customization = useSelector((state) => state.customization)
@@ -76,7 +77,31 @@ const UpsertResultDialog = ({ show, dialogProps, onCancel }) => {
                 </>
             </DialogContent>
             <DialogActions>
-                <Button onClick={onCancel}>Close</Button>
+                {dialogProps.goToRetrievalQuery && (
+                    <div style={{ display: 'flex', flexDirection: 'column', width: '100%', marginLeft: '15px', marginRight: '15px' }}>
+                        <Button
+                            variant='contained'
+                            fullWidth
+                            sx={{
+                                borderRadius: 2,
+                                height: '100%',
+                                backgroundImage: `linear-gradient(to right, #3f5efb, #fc466b)`,
+                                '&:hover': {
+                                    backgroundImage: `linear-gradient(to right, #2b4efb, #fe2752)`
+                                },
+                                mb: 2
+                            }}
+                            startIcon={<IconZoomScan />}
+                            onClick={onGoToRetrievalQuery}
+                        >
+                            Test Retrieval
+                        </Button>
+                        <Button fullWidth onClick={onCancel}>
+                            Close
+                        </Button>
+                    </div>
+                )}
+                {!dialogProps.goToRetrievalQuery && <Button onClick={onCancel}>Close</Button>}
             </DialogActions>
         </Dialog>
     ) : null
@@ -88,7 +113,7 @@ UpsertResultDialog.propTypes = {
     show: PropTypes.bool,
     dialogProps: PropTypes.object,
     onCancel: PropTypes.func,
-    onSave: PropTypes.func
+    onGoToRetrievalQuery: PropTypes.func
 }
 
 export default UpsertResultDialog