mirror of
https://github.com/farcasclaudiu/Flowise.git
synced 2026-06-28 15:00:57 +03:00
Document Store - Phase 2 (#2912)
* Document Store - Phase 2 * Adding additional columns for vector store config, document store phase 2 * Adding additional columns for vector store config, document store phase 2 * Document Store - Phase 2 - Upsert and Query * ux cleanup * retrieval settings and more ux changes * adding MMR params to execution * Making the upsert process async. * add upsert history changes * making the searchParams dynamic * removing unnecessary params * add ability to delete data from vector store * update margin for vector store query * adding option to save config in the retrieval playground * adding chunk number for query return chunks * Adding a Document Store node in the VectorStore category * update doc store status, ui touchup --------- Co-authored-by: Henry <hzj94@hotmail.com>
This commit is contained in:
Vinod Kiran
@@ -1,6 +1,5 @@
|
||||
import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
|
||||
import { DocumentStore } from '../../database/entities/DocumentStore'
|
||||
// @ts-ignore
|
||||
import {
|
||||
addSingleFileToStorage,
|
||||
getFileFromStorage,
|
||||
@@ -10,22 +9,29 @@ import {
|
||||
removeSpecificFileFromStorage
|
||||
} from 'flowise-components'
|
||||
import {
|
||||
chatType,
|
||||
DocumentStoreStatus,
|
||||
IDocumentStoreFileChunkPagedResponse,
|
||||
IDocumentStoreLoader,
|
||||
IDocumentStoreLoaderFile,
|
||||
IDocumentStoreLoaderForPreview,
|
||||
IDocumentStoreWhereUsed
|
||||
IDocumentStoreWhereUsed,
|
||||
INodeData
|
||||
} from '../../Interface'
|
||||
import { DocumentStoreFileChunk } from '../../database/entities/DocumentStoreFileChunk'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
import { databaseEntities } from '../../utils'
|
||||
import { databaseEntities, getAppVersion, saveUpsertFlowData } from '../../utils'
|
||||
import logger from '../../utils/logger'
|
||||
import nodesService from '../nodes'
|
||||
import { InternalFlowiseError } from '../../errors/internalFlowiseError'
|
||||
import { StatusCodes } from 'http-status-codes'
|
||||
import { getErrorMessage } from '../../errors/utils'
|
||||
import { ChatFlow } from '../../database/entities/ChatFlow'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { App } from '../../index'
|
||||
import { UpsertHistory } from '../../database/entities/UpsertHistory'
|
||||
import { cloneDeep, omit } from 'lodash'
|
||||
import telemetryService from '../telemetry'
|
||||
|
||||
const DOCUMENT_STORE_BASE_FOLDER = 'docustore'
|
||||
|
||||
@@ -234,8 +240,16 @@ const deleteDocumentStore = async (storeId: string) => {
|
||||
const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({
|
||||
id: storeId
|
||||
})
|
||||
if (!entity) throw new Error(`Document store ${storeId} not found`)
|
||||
if (!entity) {
|
||||
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document store ${storeId} not found`)
|
||||
}
|
||||
await removeFilesFromStorage(DOCUMENT_STORE_BASE_FOLDER, entity.id)
|
||||
|
||||
// delete upsert history
|
||||
await appServer.AppDataSource.getRepository(UpsertHistory).delete({
|
||||
chatflowid: storeId
|
||||
})
|
||||
|
||||
// now delete the store
|
||||
const tbd = await appServer.AppDataSource.getRepository(DocumentStore).delete({
|
||||
id: storeId
|
||||
@@ -285,6 +299,83 @@ const deleteDocumentStoreFileChunk = async (storeId: string, docId: string, chun
|
||||
}
|
||||
}
|
||||
|
||||
const deleteVectorStoreFromStore = async (storeId: string) => {
|
||||
try {
|
||||
const appServer = getRunningExpressApp()
|
||||
const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({
|
||||
id: storeId
|
||||
})
|
||||
if (!entity) {
|
||||
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document store ${storeId} not found`)
|
||||
}
|
||||
|
||||
if (!entity.embeddingConfig) {
|
||||
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Embedding for Document store ${storeId} not found`)
|
||||
}
|
||||
|
||||
if (!entity.vectorStoreConfig) {
|
||||
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Vector Store for Document store ${storeId} not found`)
|
||||
}
|
||||
|
||||
if (!entity.recordManagerConfig) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.NOT_FOUND,
|
||||
`Record Manager for Document Store ${storeId} is needed to delete data from Vector Store`
|
||||
)
|
||||
}
|
||||
|
||||
const options: ICommonObject = {
|
||||
chatflowid: storeId,
|
||||
appDataSource: appServer.AppDataSource,
|
||||
databaseEntities,
|
||||
logger
|
||||
}
|
||||
|
||||
// Get Record Manager Instance
|
||||
const recordManagerConfig = JSON.parse(entity.recordManagerConfig)
|
||||
const recordManagerObj = await _createRecordManagerObject(
|
||||
appServer,
|
||||
{ recordManagerName: recordManagerConfig.name, recordManagerConfig: recordManagerConfig.config },
|
||||
options
|
||||
)
|
||||
|
||||
// Get Embeddings Instance
|
||||
const embeddingConfig = JSON.parse(entity.embeddingConfig)
|
||||
const embeddingObj = await _createEmbeddingsObject(
|
||||
appServer,
|
||||
{ embeddingName: embeddingConfig.name, embeddingConfig: embeddingConfig.config },
|
||||
options
|
||||
)
|
||||
|
||||
// Get Vector Store Node Data
|
||||
const vectorStoreConfig = JSON.parse(entity.vectorStoreConfig)
|
||||
const vStoreNodeData = _createVectorStoreNodeData(
|
||||
appServer,
|
||||
{ vectorStoreName: vectorStoreConfig.name, vectorStoreConfig: vectorStoreConfig.config },
|
||||
embeddingObj,
|
||||
recordManagerObj
|
||||
)
|
||||
|
||||
// Get Vector Store Instance
|
||||
const vectorStoreObj = await _createVectorStoreObject(
|
||||
appServer,
|
||||
{ vectorStoreName: vectorStoreConfig.name, vectorStoreConfig: vectorStoreConfig.config },
|
||||
vStoreNodeData
|
||||
)
|
||||
const idsToDelete: string[] = [] // empty ids because we get it dynamically from the record manager
|
||||
|
||||
// Call the delete method of the vector store
|
||||
if (vectorStoreObj.vectorStoreMethods.delete) {
|
||||
await vectorStoreObj.vectorStoreMethods.delete(vStoreNodeData, idsToDelete, options)
|
||||
}
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: documentStoreServices.deleteVectorStoreFromStore - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const editDocumentStoreFileChunk = async (storeId: string, docId: string, chunkId: string, content: string, metadata: ICommonObject) => {
|
||||
try {
|
||||
const appServer = getRunningExpressApp()
|
||||
@@ -700,6 +791,417 @@ const updateDocumentStoreUsage = async (chatId: string, storeId: string | undefi
|
||||
}
|
||||
}
|
||||
|
||||
const updateVectorStoreConfigOnly = async (data: ICommonObject) => {
|
||||
try {
|
||||
const appServer = getRunningExpressApp()
|
||||
const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({
|
||||
id: data.storeId
|
||||
})
|
||||
if (!entity) {
|
||||
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document store ${data.storeId} not found`)
|
||||
}
|
||||
|
||||
if (data.vectorStoreName) {
|
||||
entity.vectorStoreConfig = JSON.stringify({
|
||||
config: data.vectorStoreConfig,
|
||||
name: data.vectorStoreName
|
||||
})
|
||||
|
||||
const updatedEntity = await appServer.AppDataSource.getRepository(DocumentStore).save(entity)
|
||||
return updatedEntity
|
||||
}
|
||||
return {}
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: documentStoreServices.updateVectorStoreConfig - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
const saveVectorStoreConfig = async (data: ICommonObject) => {
|
||||
try {
|
||||
const appServer = getRunningExpressApp()
|
||||
const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({
|
||||
id: data.storeId
|
||||
})
|
||||
if (!entity) {
|
||||
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document store ${data.storeId} not found`)
|
||||
}
|
||||
|
||||
if (data.embeddingName) {
|
||||
entity.embeddingConfig = JSON.stringify({
|
||||
config: data.embeddingConfig,
|
||||
name: data.embeddingName
|
||||
})
|
||||
} else if (!data.embeddingName && !data.embeddingConfig) {
|
||||
entity.embeddingConfig = null
|
||||
}
|
||||
|
||||
if (data.vectorStoreName) {
|
||||
entity.vectorStoreConfig = JSON.stringify({
|
||||
config: data.vectorStoreConfig,
|
||||
name: data.vectorStoreName
|
||||
})
|
||||
} else if (!data.vectorStoreName && !data.vectorStoreConfig) {
|
||||
entity.vectorStoreConfig = null
|
||||
}
|
||||
|
||||
if (data.recordManagerName) {
|
||||
entity.recordManagerConfig = JSON.stringify({
|
||||
config: data.recordManagerConfig,
|
||||
name: data.recordManagerName
|
||||
})
|
||||
} else if (!data.recordManagerName && !data.recordManagerConfig) {
|
||||
entity.recordManagerConfig = null
|
||||
}
|
||||
|
||||
if (entity.status !== DocumentStoreStatus.UPSERTED && (data.vectorStoreName || data.recordManagerName || data.embeddingName)) {
|
||||
// if the store is not already in sync, mark it as sync
|
||||
// this also means that the store is not yet sync'ed to vector store
|
||||
entity.status = DocumentStoreStatus.SYNC
|
||||
}
|
||||
await appServer.AppDataSource.getRepository(DocumentStore).save(entity)
|
||||
return entity
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: documentStoreServices.saveVectorStoreConfig - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const insertIntoVectorStore = async (data: ICommonObject) => {
|
||||
try {
|
||||
const appServer = getRunningExpressApp()
|
||||
const entity = await saveVectorStoreConfig(data)
|
||||
entity.status = DocumentStoreStatus.UPSERTING
|
||||
await appServer.AppDataSource.getRepository(DocumentStore).save(entity)
|
||||
|
||||
// TODO: to be moved into a worker thread...
|
||||
const indexResult = await _insertIntoVectorStoreWorkerThread(data)
|
||||
return indexResult
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: documentStoreServices.insertIntoVectorStore - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const _insertIntoVectorStoreWorkerThread = async (data: ICommonObject) => {
|
||||
try {
|
||||
const appServer = getRunningExpressApp()
|
||||
const entity = await saveVectorStoreConfig(data)
|
||||
let upsertHistory: Record<string, any> = {}
|
||||
const chatflowid = data.storeId // fake chatflowid because this is not tied to any chatflow
|
||||
|
||||
const options: ICommonObject = {
|
||||
chatflowid,
|
||||
appDataSource: appServer.AppDataSource,
|
||||
databaseEntities,
|
||||
logger
|
||||
}
|
||||
|
||||
let recordManagerObj = undefined
|
||||
|
||||
// Get Record Manager Instance
|
||||
if (data.recordManagerName && data.recordManagerConfig) {
|
||||
recordManagerObj = await _createRecordManagerObject(appServer, data, options, upsertHistory)
|
||||
}
|
||||
|
||||
// Get Embeddings Instance
|
||||
const embeddingObj = await _createEmbeddingsObject(appServer, data, options, upsertHistory)
|
||||
|
||||
// Get Vector Store Node Data
|
||||
const vStoreNodeData = _createVectorStoreNodeData(appServer, data, embeddingObj, recordManagerObj)
|
||||
// Prepare docs for upserting
|
||||
const chunks = await appServer.AppDataSource.getRepository(DocumentStoreFileChunk).find({
|
||||
where: {
|
||||
storeId: data.storeId
|
||||
}
|
||||
})
|
||||
const docs: Document[] = chunks.map((chunk: DocumentStoreFileChunk) => {
|
||||
return new Document({
|
||||
pageContent: chunk.pageContent,
|
||||
metadata: JSON.parse(chunk.metadata)
|
||||
})
|
||||
})
|
||||
vStoreNodeData.inputs.document = docs
|
||||
|
||||
// Get Vector Store Instance
|
||||
const vectorStoreObj = await _createVectorStoreObject(appServer, data, vStoreNodeData, upsertHistory)
|
||||
const indexResult = await vectorStoreObj.vectorStoreMethods.upsert(vStoreNodeData, options)
|
||||
|
||||
// Save to DB
|
||||
if (indexResult) {
|
||||
const result = cloneDeep(upsertHistory)
|
||||
result['flowData'] = JSON.stringify(result['flowData'])
|
||||
result['result'] = JSON.stringify(omit(indexResult, ['totalKeys', 'addedDocs']))
|
||||
result.chatflowid = chatflowid
|
||||
const newUpsertHistory = new UpsertHistory()
|
||||
Object.assign(newUpsertHistory, result)
|
||||
const upsertHistoryItem = appServer.AppDataSource.getRepository(UpsertHistory).create(newUpsertHistory)
|
||||
await appServer.AppDataSource.getRepository(UpsertHistory).save(upsertHistoryItem)
|
||||
}
|
||||
|
||||
await telemetryService.createEvent({
|
||||
name: `vector_upserted`,
|
||||
data: {
|
||||
version: await getAppVersion(),
|
||||
chatlowId: chatflowid,
|
||||
type: chatType.INTERNAL,
|
||||
flowGraph: omit(indexResult['result'], ['totalKeys', 'addedDocs'])
|
||||
}
|
||||
})
|
||||
|
||||
entity.status = DocumentStoreStatus.UPSERTED
|
||||
await appServer.AppDataSource.getRepository(DocumentStore).save(entity)
|
||||
|
||||
return indexResult ?? { result: 'Successfully Upserted' }
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: documentStoreServices._insertIntoVectorStoreWorkerThread - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Get all component nodes - Embeddings
|
||||
const getEmbeddingProviders = async () => {
|
||||
try {
|
||||
const dbResponse = await nodesService.getAllNodesForCategory('Embeddings')
|
||||
return dbResponse.filter((node) => !node.tags?.includes('LlamaIndex'))
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: documentStoreServices.getEmbeddingProviders - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Get all component nodes - Vector Stores
|
||||
const getVectorStoreProviders = async () => {
|
||||
try {
|
||||
const dbResponse = await nodesService.getAllNodesForCategory('Vector Stores')
|
||||
return dbResponse.filter((node) => !node.tags?.includes('LlamaIndex') && node.name !== 'documentStoreVS')
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: documentStoreServices.getVectorStoreProviders - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
// Get all component nodes - Vector Stores
|
||||
const getRecordManagerProviders = async () => {
|
||||
try {
|
||||
const dbResponse = await nodesService.getAllNodesForCategory('Record Manager')
|
||||
return dbResponse.filter((node) => !node.tags?.includes('LlamaIndex'))
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: documentStoreServices.getRecordManagerProviders - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const queryVectorStore = async (data: ICommonObject) => {
|
||||
try {
|
||||
const appServer = getRunningExpressApp()
|
||||
const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({
|
||||
id: data.storeId
|
||||
})
|
||||
if (!entity) {
|
||||
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Document store ${data.storeId} not found`)
|
||||
}
|
||||
const options: ICommonObject = {
|
||||
chatflowid: uuidv4(),
|
||||
appDataSource: appServer.AppDataSource,
|
||||
databaseEntities,
|
||||
logger
|
||||
}
|
||||
|
||||
if (!entity.embeddingConfig) {
|
||||
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Embedding for ${data.storeId} is not configured`)
|
||||
}
|
||||
|
||||
if (!entity.vectorStoreConfig) {
|
||||
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Vector Store for ${data.storeId} is not configured`)
|
||||
}
|
||||
|
||||
const embeddingConfig = JSON.parse(entity.embeddingConfig)
|
||||
data.embeddingName = embeddingConfig.name
|
||||
data.embeddingConfig = embeddingConfig.config
|
||||
let embeddingObj = await _createEmbeddingsObject(appServer, data, options)
|
||||
|
||||
const vsConfig = JSON.parse(entity.vectorStoreConfig)
|
||||
data.vectorStoreName = vsConfig.name
|
||||
data.vectorStoreConfig = vsConfig.config
|
||||
if (data.inputs) {
|
||||
data.vectorStoreConfig = { ...vsConfig.config, ...data.inputs }
|
||||
}
|
||||
|
||||
const vStoreNodeData = _createVectorStoreNodeData(appServer, data, embeddingObj, undefined)
|
||||
|
||||
// Get Vector Store Instance
|
||||
const vectorStoreObj = await _createVectorStoreObject(appServer, data, vStoreNodeData)
|
||||
const retriever = await vectorStoreObj.init(vStoreNodeData, '', options)
|
||||
if (!retriever) {
|
||||
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Failed to create retriever`)
|
||||
}
|
||||
const startMillis = Date.now()
|
||||
const results = await retriever.invoke(data.query, undefined)
|
||||
if (!results) {
|
||||
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Failed to retrieve results`)
|
||||
}
|
||||
const endMillis = Date.now()
|
||||
const timeTaken = endMillis - startMillis
|
||||
const docs: any = results.map((result: IDocument) => {
|
||||
return {
|
||||
pageContent: result.pageContent,
|
||||
metadata: result.metadata,
|
||||
id: uuidv4()
|
||||
}
|
||||
})
|
||||
// query our document store chunk with the storeId and pageContent
|
||||
for (const doc of docs) {
|
||||
const documentStoreChunk = await appServer.AppDataSource.getRepository(DocumentStoreFileChunk).findOneBy({
|
||||
storeId: data.storeId,
|
||||
pageContent: doc.pageContent
|
||||
})
|
||||
if (documentStoreChunk) {
|
||||
doc.id = documentStoreChunk.id
|
||||
doc.chunkNo = documentStoreChunk.chunkNo
|
||||
} else {
|
||||
// this should not happen, only possible if the vector store has more content
|
||||
// than our document store
|
||||
doc.id = uuidv4()
|
||||
doc.chunkNo = -1
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
timeTaken: timeTaken,
|
||||
docs: docs
|
||||
}
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: documentStoreServices.queryVectorStore - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const _createEmbeddingsObject = async (
|
||||
appServer: App,
|
||||
data: ICommonObject,
|
||||
options: ICommonObject,
|
||||
upsertHistory?: Record<string, any>
|
||||
): Promise<any> => {
|
||||
// prepare embedding node data
|
||||
const embeddingComponent = appServer.nodesPool.componentNodes[data.embeddingName]
|
||||
const embeddingNodeData: any = {
|
||||
inputs: { ...data.embeddingConfig },
|
||||
outputs: { output: 'document' },
|
||||
id: `${embeddingComponent.name}_0`,
|
||||
label: embeddingComponent.label,
|
||||
name: embeddingComponent.name,
|
||||
category: embeddingComponent.category,
|
||||
inputParams: embeddingComponent.inputs || []
|
||||
}
|
||||
if (data.embeddingConfig.credential) {
|
||||
embeddingNodeData.credential = data.embeddingConfig.credential
|
||||
}
|
||||
|
||||
// save to upsert history
|
||||
if (upsertHistory) upsertHistory['flowData'] = saveUpsertFlowData(embeddingNodeData, upsertHistory)
|
||||
|
||||
// init embedding object
|
||||
const embeddingNodeInstanceFilePath = embeddingComponent.filePath as string
|
||||
const embeddingNodeModule = await import(embeddingNodeInstanceFilePath)
|
||||
const embeddingNodeInstance = new embeddingNodeModule.nodeClass()
|
||||
const embeddingObj = await embeddingNodeInstance.init(embeddingNodeData, '', options)
|
||||
if (!embeddingObj) {
|
||||
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Failed to create EmbeddingObj`)
|
||||
}
|
||||
return embeddingObj
|
||||
}
|
||||
|
||||
const _createRecordManagerObject = async (
|
||||
appServer: App,
|
||||
data: ICommonObject,
|
||||
options: ICommonObject,
|
||||
upsertHistory?: Record<string, any>
|
||||
) => {
|
||||
// prepare record manager node data
|
||||
const recordManagerComponent = appServer.nodesPool.componentNodes[data.recordManagerName]
|
||||
const rmNodeData: any = {
|
||||
inputs: { ...data.recordManagerConfig },
|
||||
id: `${recordManagerComponent.name}_0`,
|
||||
inputParams: recordManagerComponent.inputs,
|
||||
label: recordManagerComponent.label,
|
||||
name: recordManagerComponent.name,
|
||||
category: recordManagerComponent.category
|
||||
}
|
||||
if (data.recordManagerConfig.credential) {
|
||||
rmNodeData.credential = data.recordManagerConfig.credential
|
||||
}
|
||||
|
||||
// save to upsert history
|
||||
if (upsertHistory) upsertHistory['flowData'] = saveUpsertFlowData(rmNodeData, upsertHistory)
|
||||
|
||||
// init record manager object
|
||||
const rmNodeInstanceFilePath = recordManagerComponent.filePath as string
|
||||
const rmNodeModule = await import(rmNodeInstanceFilePath)
|
||||
const rmNodeInstance = new rmNodeModule.nodeClass()
|
||||
const recordManagerObj = await rmNodeInstance.init(rmNodeData, '', options)
|
||||
if (!recordManagerObj) {
|
||||
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Failed to create RecordManager obj`)
|
||||
}
|
||||
return recordManagerObj
|
||||
}
|
||||
|
||||
const _createVectorStoreNodeData = (appServer: App, data: ICommonObject, embeddingObj: any, recordManagerObj?: any) => {
|
||||
const vectorStoreComponent = appServer.nodesPool.componentNodes[data.vectorStoreName]
|
||||
const vStoreNodeData: any = {
|
||||
id: `${vectorStoreComponent.name}_0`,
|
||||
inputs: { ...data.vectorStoreConfig },
|
||||
outputs: { output: 'retriever' },
|
||||
label: vectorStoreComponent.label,
|
||||
name: vectorStoreComponent.name,
|
||||
category: vectorStoreComponent.category
|
||||
}
|
||||
if (data.vectorStoreConfig.credential) {
|
||||
vStoreNodeData.credential = data.vectorStoreConfig.credential
|
||||
}
|
||||
|
||||
if (embeddingObj) {
|
||||
vStoreNodeData.inputs.embeddings = embeddingObj
|
||||
}
|
||||
|
||||
if (recordManagerObj) {
|
||||
vStoreNodeData.inputs.recordManager = recordManagerObj
|
||||
}
|
||||
|
||||
// Get all input params except the ones that are anchor points to avoid JSON stringify circular error
|
||||
const filterInputParams = ['document', 'embeddings', 'recordManager']
|
||||
const inputParams = vectorStoreComponent.inputs?.filter((input) => !filterInputParams.includes(input.name))
|
||||
vStoreNodeData.inputParams = inputParams
|
||||
return vStoreNodeData
|
||||
}
|
||||
|
||||
const _createVectorStoreObject = async (
|
||||
appServer: App,
|
||||
data: ICommonObject,
|
||||
vStoreNodeData: INodeData,
|
||||
upsertHistory?: Record<string, any>
|
||||
) => {
|
||||
const vStoreNodeInstanceFilePath = appServer.nodesPool.componentNodes[data.vectorStoreName].filePath as string
|
||||
const vStoreNodeModule = await import(vStoreNodeInstanceFilePath)
|
||||
const vStoreNodeInstance = new vStoreNodeModule.nodeClass()
|
||||
if (upsertHistory) upsertHistory['flowData'] = saveUpsertFlowData(vStoreNodeData, upsertHistory)
|
||||
return vStoreNodeInstance
|
||||
}
|
||||
|
||||
export default {
|
||||
updateDocumentStoreUsage,
|
||||
deleteDocumentStore,
|
||||
@@ -714,5 +1216,13 @@ export default {
|
||||
processAndSaveChunks,
|
||||
deleteDocumentStoreFileChunk,
|
||||
editDocumentStoreFileChunk,
|
||||
getDocumentLoaders
|
||||
getDocumentLoaders,
|
||||
insertIntoVectorStore,
|
||||
getEmbeddingProviders,
|
||||
getVectorStoreProviders,
|
||||
getRecordManagerProviders,
|
||||
saveVectorStoreConfig,
|
||||
queryVectorStore,
|
||||
deleteVectorStoreFromStore,
|
||||
updateVectorStoreConfigOnly
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user