diff --git a/packages/components/nodes/vectorstores/Chroma/Chroma.ts b/packages/components/nodes/vectorstores/Chroma/Chroma.ts index 385872f2..d71f0976 100644 --- a/packages/components/nodes/vectorstores/Chroma/Chroma.ts +++ b/packages/components/nodes/vectorstores/Chroma/Chroma.ts @@ -150,6 +150,42 @@ class Chroma_VectorStores implements INode { } catch (e) { throw new Error(e) } + }, + async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise { + const collectionName = nodeData.inputs?.collectionName as string + const embeddings = nodeData.inputs?.embeddings as Embeddings + const chromaURL = nodeData.inputs?.chromaURL as string + const recordManager = nodeData.inputs?.recordManager + + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const chromaApiKey = getCredentialParam('chromaApiKey', credentialData, nodeData) + + const obj: { + collectionName: string + url?: string + chromaApiKey?: string + } = { collectionName } + if (chromaURL) obj.url = chromaURL + if (chromaApiKey) obj.chromaApiKey = chromaApiKey + + try { + if (recordManager) { + const vectorStoreName = collectionName + await recordManager.createSchema() + ;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName + const keys: string[] = await recordManager.listKeys({}) + + const chromaStore = new ChromaExtended(embeddings, obj) + + await chromaStore.delete({ ids: keys }) + await recordManager.deleteKeys(keys) + } else { + const chromaStore = new ChromaExtended(embeddings, obj) + await chromaStore.delete({ ids }) + } + } catch (e) { + throw new Error(e) + } } } diff --git a/packages/components/nodes/vectorstores/DocumentStoreVS/DocStoreVector.ts b/packages/components/nodes/vectorstores/DocumentStoreVS/DocStoreVector.ts new file mode 100644 index 00000000..6ce47f71 --- /dev/null +++ b/packages/components/nodes/vectorstores/DocumentStoreVS/DocStoreVector.ts @@ -0,0 +1,174 @@ +import { ICommonObject, IDatabaseEntity, INode, INodeData, INodeOptionsValue, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { DataSource } from 'typeorm' + +class DocStore_VectorStores implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + outputs: INodeOutputsValue[] + badge: string + + constructor() { + this.label = 'Document Store (Vector)' + this.name = 'documentStoreVS' + this.version = 1.0 + this.type = 'DocumentStoreVS' + this.icon = 'dstore.svg' + this.badge = 'New' + this.category = 'Vector Stores' + this.description = `Search and retrieve documents from Document Store` + this.baseClasses = [this.type] + this.inputs = [ + { + label: 'Select Store', + name: 'selectedStore', + type: 'asyncOptions', + loadMethod: 'listStores' + } + ] + this.outputs = [ + { + label: 'Retriever', + name: 'retriever', + baseClasses: ['BaseRetriever'] + }, + { + label: 'Vector Store', + name: 'vectorStore', + baseClasses: ['VectorStore'] + } + ] + } + + //@ts-ignore + loadMethods = { + async listStores(_: INodeData, options: ICommonObject): Promise { + const returnData: INodeOptionsValue[] = [] + + const appDataSource = options.appDataSource as DataSource + const databaseEntities = options.databaseEntities as IDatabaseEntity + + if (appDataSource === undefined || !appDataSource) { + return returnData + } + + const stores = await appDataSource.getRepository(databaseEntities['DocumentStore']).find() + for (const store of stores) { + if (store.status === 'UPSERTED') { + const obj = { + name: store.id, + label: store.name, + description: store.description + } + returnData.push(obj) + } + } + return returnData + } + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const selectedStore = nodeData.inputs?.selectedStore as string + const appDataSource = options.appDataSource as DataSource + const databaseEntities = options.databaseEntities as IDatabaseEntity + const output = nodeData.outputs?.output as string + + const entity = await appDataSource.getRepository(databaseEntities['DocumentStore']).findOneBy({ id: selectedStore }) + if (!entity) { + return { error: 'Store not found' } + } + const data: ICommonObject = {} + data.output = output + + // Prepare Embeddings Instance + const embeddingConfig = JSON.parse(entity.embeddingConfig) + data.embeddingName = embeddingConfig.name + data.embeddingConfig = embeddingConfig.config + let embeddingObj = await _createEmbeddingsObject(options.componentNodes, data, options) + if (!embeddingObj) { + return { error: 'Failed to create EmbeddingObj' } + } + + // Prepare Vector Store Instance + const vsConfig = JSON.parse(entity.vectorStoreConfig) + data.vectorStoreName = vsConfig.name + data.vectorStoreConfig = vsConfig.config + if (data.inputs) { + data.vectorStoreConfig = { ...vsConfig.config, ...data.inputs } + } + + // Prepare Vector Store Node Data + const vStoreNodeData = _createVectorStoreNodeData(options.componentNodes, data, embeddingObj) + + // Finally create the Vector Store or Retriever object (data.output) + const vectorStoreObj = await _createVectorStoreObject(options.componentNodes, data) + const retrieverOrVectorStore = await vectorStoreObj.init(vStoreNodeData, '', options) + if (!retrieverOrVectorStore) { + return { error: 'Failed to create vectorStore' } + } + return retrieverOrVectorStore + } +} + +const _createEmbeddingsObject = async (componentNodes: ICommonObject, data: ICommonObject, options: ICommonObject): Promise => { + // prepare embedding node data + const embeddingComponent = componentNodes[data.embeddingName] + const embeddingNodeData: any = { + inputs: { ...data.embeddingConfig }, + outputs: { output: 'document' }, + id: `${embeddingComponent.name}_0`, + label: embeddingComponent.label, + name: embeddingComponent.name, + category: embeddingComponent.category, + inputParams: embeddingComponent.inputs || [] + } + if (data.embeddingConfig.credential) { + embeddingNodeData.credential = data.embeddingConfig.credential + } + + // init embedding object + const embeddingNodeInstanceFilePath = embeddingComponent.filePath as string + const embeddingNodeModule = await import(embeddingNodeInstanceFilePath) + const embeddingNodeInstance = new embeddingNodeModule.nodeClass() + return await embeddingNodeInstance.init(embeddingNodeData, '', options) +} + +const _createVectorStoreNodeData = (componentNodes: ICommonObject, data: ICommonObject, embeddingObj: any) => { + const vectorStoreComponent = componentNodes[data.vectorStoreName] + const vStoreNodeData: any = { + id: `${vectorStoreComponent.name}_0`, + inputs: { ...data.vectorStoreConfig }, + outputs: { output: data.output }, + label: vectorStoreComponent.label, + name: vectorStoreComponent.name, + category: vectorStoreComponent.category + } + if (data.vectorStoreConfig.credential) { + vStoreNodeData.credential = data.vectorStoreConfig.credential + } + + if (embeddingObj) { + vStoreNodeData.inputs.embeddings = embeddingObj + } + + // Get all input params except the ones that are anchor points to avoid JSON stringify circular error + const filterInputParams = ['document', 'embeddings', 'recordManager'] + const inputParams = vectorStoreComponent.inputs?.filter((input: any) => !filterInputParams.includes(input.name)) + vStoreNodeData.inputParams = inputParams + return vStoreNodeData +} + +const _createVectorStoreObject = async (componentNodes: ICommonObject, data: ICommonObject) => { + const vStoreNodeInstanceFilePath = componentNodes[data.vectorStoreName].filePath as string + const vStoreNodeModule = await import(vStoreNodeInstanceFilePath) + const vStoreNodeInstance = new vStoreNodeModule.nodeClass() + return vStoreNodeInstance +} + +module.exports = { nodeClass: DocStore_VectorStores } diff --git a/packages/components/nodes/vectorstores/DocumentStoreVS/dstore.svg b/packages/components/nodes/vectorstores/DocumentStoreVS/dstore.svg new file mode 100644 index 00000000..4c9812c4 --- /dev/null +++ b/packages/components/nodes/vectorstores/DocumentStoreVS/dstore.svg @@ -0,0 +1,15 @@ + + + + + \ No newline at end of file diff --git a/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch.ts b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch.ts index 6271d972..d752928e 100644 --- a/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch.ts +++ b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch.ts @@ -163,6 +163,35 @@ class Elasticsearch_VectorStores implements INode { } catch (e) { throw new Error(e) } + }, + async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise { + const indexName = nodeData.inputs?.indexName as string + const embeddings = nodeData.inputs?.embeddings as Embeddings + const similarityMeasure = nodeData.inputs?.similarityMeasure as string + const recordManager = nodeData.inputs?.recordManager + + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const endPoint = getCredentialParam('endpoint', credentialData, nodeData) + const cloudId = getCredentialParam('cloudId', credentialData, nodeData) + + const elasticSearchClientArgs = prepareClientArgs(endPoint, cloudId, credentialData, nodeData, similarityMeasure, indexName) + const vectorStore = new ElasticVectorSearch(embeddings, elasticSearchClientArgs) + + try { + if (recordManager) { + const vectorStoreName = indexName + await recordManager.createSchema() + ;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName + const keys: string[] = await recordManager.listKeys({}) + + await vectorStore.delete({ ids: keys }) + await recordManager.deleteKeys(keys) + } else { + await vectorStore.delete({ ids }) + } + } catch (e) { + throw new Error(e) + } } } diff --git a/packages/components/nodes/vectorstores/Pinecone/Pinecone.ts b/packages/components/nodes/vectorstores/Pinecone/Pinecone.ts index d12510fa..b7af388c 100644 --- a/packages/components/nodes/vectorstores/Pinecone/Pinecone.ts +++ b/packages/components/nodes/vectorstores/Pinecone/Pinecone.ts @@ -184,6 +184,45 @@ class Pinecone_VectorStores implements INode { } catch (e) { throw new Error(e) } + }, + async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise { + const _index = nodeData.inputs?.pineconeIndex as string + const pineconeNamespace = nodeData.inputs?.pineconeNamespace as string + const embeddings = nodeData.inputs?.embeddings as Embeddings + const pineconeTextKey = nodeData.inputs?.pineconeTextKey as string + const recordManager = nodeData.inputs?.recordManager + + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const pineconeApiKey = getCredentialParam('pineconeApiKey', credentialData, nodeData) + + const client = getPineconeClient({ apiKey: pineconeApiKey }) + + const pineconeIndex = client.Index(_index) + + const obj: PineconeStoreParams = { + pineconeIndex, + textKey: pineconeTextKey || 'text' + } + + if (pineconeNamespace) obj.namespace = pineconeNamespace + const pineconeStore = new PineconeStore(embeddings, obj) + + try { + if (recordManager) { + const vectorStoreName = pineconeNamespace + await recordManager.createSchema() + ;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName + const keys: string[] = await recordManager.listKeys({}) + + await pineconeStore.delete({ ids: keys }) + await recordManager.deleteKeys(keys) + } else { + const pineconeStore = new PineconeStore(embeddings, obj) + await pineconeStore.delete({ ids }) + } + } catch (e) { + throw new Error(e) + } } } diff --git a/packages/components/nodes/vectorstores/Postgres/Postgres.ts b/packages/components/nodes/vectorstores/Postgres/Postgres.ts index 3053d087..4a4d5443 100644 --- a/packages/components/nodes/vectorstores/Postgres/Postgres.ts +++ b/packages/components/nodes/vectorstores/Postgres/Postgres.ts @@ -201,6 +201,58 @@ class Postgres_VectorStores implements INode { } catch (e) { throw new Error(e) } + }, + async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise { + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const user = getCredentialParam('user', credentialData, nodeData) + const password = getCredentialParam('password', credentialData, nodeData) + const _tableName = nodeData.inputs?.tableName as string + const tableName = _tableName ? _tableName : 'documents' + const embeddings = nodeData.inputs?.embeddings as Embeddings + const additionalConfig = nodeData.inputs?.additionalConfig as string + const recordManager = nodeData.inputs?.recordManager + + let additionalConfiguration = {} + if (additionalConfig) { + try { + additionalConfiguration = typeof additionalConfig === 'object' ? additionalConfig : JSON.parse(additionalConfig) + } catch (exception) { + throw new Error('Invalid JSON in the Additional Configuration: ' + exception) + } + } + + const postgresConnectionOptions = { + ...additionalConfiguration, + type: 'postgres', + host: nodeData.inputs?.host as string, + port: nodeData.inputs?.port as number, + username: user, + password: password, + database: nodeData.inputs?.database as string + } + + const args = { + postgresConnectionOptions: postgresConnectionOptions as DataSourceOptions, + tableName: tableName + } + + const vectorStore = await TypeORMVectorStore.fromDataSource(embeddings, args) + + try { + if (recordManager) { + const vectorStoreName = tableName + await recordManager.createSchema() + ;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName + const keys: string[] = await recordManager.listKeys({}) + + await vectorStore.delete({ ids: keys }) + await recordManager.deleteKeys(keys) + } else { + await vectorStore.delete({ ids }) + } + } catch (e) { + throw new Error(e) + } } } diff --git a/packages/components/nodes/vectorstores/Qdrant/Qdrant.ts b/packages/components/nodes/vectorstores/Qdrant/Qdrant.ts index 0b20c74b..c12de2c7 100644 --- a/packages/components/nodes/vectorstores/Qdrant/Qdrant.ts +++ b/packages/components/nodes/vectorstores/Qdrant/Qdrant.ts @@ -291,6 +291,69 @@ class Qdrant_VectorStores implements INode { } catch (e) { throw new Error(e) } + }, + async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise { + const qdrantServerUrl = nodeData.inputs?.qdrantServerUrl as string + const collectionName = nodeData.inputs?.qdrantCollection as string + const embeddings = nodeData.inputs?.embeddings as Embeddings + const qdrantSimilarity = nodeData.inputs?.qdrantSimilarity + const qdrantVectorDimension = nodeData.inputs?.qdrantVectorDimension + const recordManager = nodeData.inputs?.recordManager + + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const qdrantApiKey = getCredentialParam('qdrantApiKey', credentialData, nodeData) + + const port = Qdrant_VectorStores.determinePortByUrl(qdrantServerUrl) + + const client = new QdrantClient({ + url: qdrantServerUrl, + apiKey: qdrantApiKey, + port: port + }) + + const dbConfig: QdrantLibArgs = { + client, + url: qdrantServerUrl, + collectionName, + collectionConfig: { + vectors: { + size: qdrantVectorDimension ? parseInt(qdrantVectorDimension, 10) : 1536, + distance: qdrantSimilarity ?? 'Cosine' + } + } + } + + const vectorStore = new QdrantVectorStore(embeddings, dbConfig) + + vectorStore.delete = async (params: { ids: string[] }): Promise => { + const { ids } = params + + if (ids?.length) { + try { + client.delete(collectionName, { + points: ids + }) + } catch (e) { + console.error('Failed to delete') + } + } + } + + try { + if (recordManager) { + const vectorStoreName = collectionName + await recordManager.createSchema() + ;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName + const keys: string[] = await recordManager.listKeys({}) + + await vectorStore.delete({ ids: keys }) + await recordManager.deleteKeys(keys) + } else { + await vectorStore.delete({ ids }) + } + } catch (e) { + throw new Error(e) + } } } diff --git a/packages/components/nodes/vectorstores/Supabase/Supabase.ts b/packages/components/nodes/vectorstores/Supabase/Supabase.ts index 2c01c4a8..25f37924 100644 --- a/packages/components/nodes/vectorstores/Supabase/Supabase.ts +++ b/packages/components/nodes/vectorstores/Supabase/Supabase.ts @@ -171,6 +171,40 @@ class Supabase_VectorStores implements INode { } catch (e) { throw new Error(e) } + }, + async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise { + const supabaseProjUrl = nodeData.inputs?.supabaseProjUrl as string + const tableName = nodeData.inputs?.tableName as string + const queryName = nodeData.inputs?.queryName as string + const embeddings = nodeData.inputs?.embeddings as Embeddings + const recordManager = nodeData.inputs?.recordManager + + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const supabaseApiKey = getCredentialParam('supabaseApiKey', credentialData, nodeData) + + const client = createClient(supabaseProjUrl, supabaseApiKey) + + const supabaseStore = new SupabaseVectorStore(embeddings, { + client, + tableName: tableName, + queryName: queryName + }) + + try { + if (recordManager) { + const vectorStoreName = tableName + '_' + queryName + await recordManager.createSchema() + ;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName + const keys: string[] = await recordManager.listKeys({}) + + await supabaseStore.delete({ ids: keys }) + await recordManager.deleteKeys(keys) + } else { + await supabaseStore.delete({ ids }) + } + } catch (e) { + throw new Error(e) + } } } diff --git a/packages/components/nodes/vectorstores/Upstash/Upstash.ts b/packages/components/nodes/vectorstores/Upstash/Upstash.ts index 9958b4f4..e41ab849 100644 --- a/packages/components/nodes/vectorstores/Upstash/Upstash.ts +++ b/packages/components/nodes/vectorstores/Upstash/Upstash.ts @@ -145,6 +145,41 @@ class Upstash_VectorStores implements INode { } catch (e) { throw new Error(e) } + }, + async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise { + const embeddings = nodeData.inputs?.embeddings as Embeddings + const recordManager = nodeData.inputs?.recordManager + + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const UPSTASH_VECTOR_REST_URL = getCredentialParam('UPSTASH_VECTOR_REST_URL', credentialData, nodeData) + const UPSTASH_VECTOR_REST_TOKEN = getCredentialParam('UPSTASH_VECTOR_REST_TOKEN', credentialData, nodeData) + + const upstashIndex = new UpstashIndex({ + url: UPSTASH_VECTOR_REST_URL, + token: UPSTASH_VECTOR_REST_TOKEN + }) + + const obj = { + index: upstashIndex + } + + const upstashStore = new UpstashVectorStore(embeddings, obj) + + try { + if (recordManager) { + const vectorStoreName = UPSTASH_VECTOR_REST_URL + await recordManager.createSchema() + ;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName + const keys: string[] = await recordManager.listKeys({}) + + await upstashStore.delete({ ids: keys }) + await recordManager.deleteKeys(keys) + } else { + await upstashStore.delete({ ids }) + } + } catch (e) { + throw new Error(e) + } } } diff --git a/packages/components/nodes/vectorstores/Weaviate/Weaviate.ts b/packages/components/nodes/vectorstores/Weaviate/Weaviate.ts index cb9799c7..85e66fb4 100644 --- a/packages/components/nodes/vectorstores/Weaviate/Weaviate.ts +++ b/packages/components/nodes/vectorstores/Weaviate/Weaviate.ts @@ -200,6 +200,53 @@ class Weaviate_VectorStores implements INode { } catch (e) { throw new Error(e) } + }, + async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise { + const weaviateScheme = nodeData.inputs?.weaviateScheme as string + const weaviateHost = nodeData.inputs?.weaviateHost as string + const weaviateIndex = nodeData.inputs?.weaviateIndex as string + const weaviateTextKey = nodeData.inputs?.weaviateTextKey as string + const weaviateMetadataKeys = nodeData.inputs?.weaviateMetadataKeys as string + const embeddings = nodeData.inputs?.embeddings as Embeddings + const recordManager = nodeData.inputs?.recordManager + + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const weaviateApiKey = getCredentialParam('weaviateApiKey', credentialData, nodeData) + + const clientConfig: any = { + scheme: weaviateScheme, + host: weaviateHost + } + if (weaviateApiKey) clientConfig.apiKey = new ApiKey(weaviateApiKey) + + const client: WeaviateClient = weaviate.client(clientConfig) + + const obj: WeaviateLibArgs = { + //@ts-ignore + client, + indexName: weaviateIndex + } + + if (weaviateTextKey) obj.textKey = weaviateTextKey + if (weaviateMetadataKeys) obj.metadataKeys = JSON.parse(weaviateMetadataKeys.replace(/\s/g, '')) + + const weaviateStore = new WeaviateStore(embeddings, obj) + + try { + if (recordManager) { + const vectorStoreName = weaviateTextKey ? weaviateIndex + '_' + weaviateTextKey : weaviateIndex + await recordManager.createSchema() + ;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName + const keys: string[] = await recordManager.listKeys({}) + + await weaviateStore.delete({ ids: keys }) + await recordManager.deleteKeys(keys) + } else { + await weaviateStore.delete({ ids }) + } + } catch (e) { + throw new Error(e) + } } } diff --git a/packages/components/src/Interface.ts b/packages/components/src/Interface.ts index a350d43f..6b687fc7 100644 --- a/packages/components/src/Interface.ts +++ b/packages/components/src/Interface.ts @@ -130,7 +130,7 @@ export interface INode extends INodeProperties { vectorStoreMethods?: { upsert: (nodeData: INodeData, options?: ICommonObject) => Promise search: (nodeData: INodeData, options?: ICommonObject) => Promise - delete: (nodeData: INodeData, options?: ICommonObject) => Promise + delete: (nodeData: INodeData, ids: string[], options?: ICommonObject) => Promise } init?(nodeData: INodeData, input: string, options?: ICommonObject): Promise run?(nodeData: INodeData, input: string, options?: ICommonObject): Promise diff --git a/packages/server/src/Interface.DocumentStore.ts b/packages/server/src/Interface.DocumentStore.ts index 837b2f4d..e319fa74 100644 --- a/packages/server/src/Interface.DocumentStore.ts +++ b/packages/server/src/Interface.DocumentStore.ts @@ -5,7 +5,9 @@ export enum DocumentStoreStatus { SYNC = 'SYNC', SYNCING = 'SYNCING', STALE = 'STALE', - NEW = 'NEW' + NEW = 'NEW', + UPSERTING = 'UPSERTING', + UPSERTED = 'UPSERTED' } export interface IDocumentStore { @@ -17,6 +19,9 @@ export interface IDocumentStore { updatedDate: Date createdDate: Date status: DocumentStoreStatus + vectorStoreConfig: string | null // JSON string + embeddingConfig: string | null // JSON string + recordManagerConfig: string | null // JSON string } export interface IDocumentStoreFileChunk { @@ -89,6 +94,9 @@ export class DocumentStoreDTO { totalChars: number chunkSize: number loaders: IDocumentStoreLoader[] + vectorStoreConfig: any + embeddingConfig: any + recordManagerConfig: any constructor() {} @@ -109,6 +117,16 @@ export class DocumentStoreDTO { documentStoreDTO.whereUsed = [] } + if (entity.vectorStoreConfig) { + documentStoreDTO.vectorStoreConfig = JSON.parse(entity.vectorStoreConfig) + } + if (entity.embeddingConfig) { + documentStoreDTO.embeddingConfig = JSON.parse(entity.embeddingConfig) + } + if (entity.recordManagerConfig) { + documentStoreDTO.recordManagerConfig = JSON.parse(entity.recordManagerConfig) + } + if (entity.loaders) { documentStoreDTO.loaders = JSON.parse(entity.loaders) documentStoreDTO.loaders.map((loader) => { diff --git a/packages/server/src/controllers/documentstore/index.ts b/packages/server/src/controllers/documentstore/index.ts index 073af3af..10041359 100644 --- a/packages/server/src/controllers/documentstore/index.ts +++ b/packages/server/src/controllers/documentstore/index.ts @@ -248,6 +248,100 @@ const getDocumentLoaders = async (req: Request, res: Response, next: NextFunctio } } +const insertIntoVectorStore = async (req: Request, res: Response, next: NextFunction) => { + try { + if (typeof req.body === 'undefined') { + throw new Error('Error: documentStoreController.insertIntoVectorStore - body not provided!') + } + const body = req.body + const apiResponse = await documentStoreService.insertIntoVectorStore(body) + return res.json(DocumentStoreDTO.fromEntity(apiResponse)) + } catch (error) { + next(error) + } +} + +const queryVectorStore = async (req: Request, res: Response, next: NextFunction) => { + try { + if (typeof req.body === 'undefined') { + throw new Error('Error: documentStoreController.queryVectorStore - body not provided!') + } + const body = req.body + const apiResponse = await documentStoreService.queryVectorStore(body) + return res.json(apiResponse) + } catch (error) { + next(error) + } +} + +const deleteVectorStoreFromStore = async (req: Request, res: Response, next: NextFunction) => { + try { + if (typeof req.params.storeId === 'undefined' || req.params.storeId === '') { + throw new InternalFlowiseError( + StatusCodes.PRECONDITION_FAILED, + `Error: documentStoreController.deleteVectorStoreFromStore - storeId not provided!` + ) + } + const apiResponse = await documentStoreService.deleteVectorStoreFromStore(req.params.storeId) + return res.json(apiResponse) + } catch (error) { + next(error) + } +} + +const saveVectorStoreConfig = async (req: Request, res: Response, next: NextFunction) => { + try { + if (typeof req.body === 'undefined') { + throw new Error('Error: documentStoreController.saveVectorStoreConfig - body not provided!') + } + const body = req.body + const apiResponse = await documentStoreService.saveVectorStoreConfig(body) + return res.json(apiResponse) + } catch (error) { + next(error) + } +} + +const updateVectorStoreConfigOnly = async (req: Request, res: Response, next: NextFunction) => { + try { + if (typeof req.body === 'undefined') { + throw new Error('Error: documentStoreController.updateVectorStoreConfigOnly - body not provided!') + } + const body = req.body + const apiResponse = await documentStoreService.updateVectorStoreConfigOnly(body) + return res.json(apiResponse) + } catch (error) { + next(error) + } +} + +const getEmbeddingProviders = async (req: Request, res: Response, next: NextFunction) => { + try { + const apiResponse = await documentStoreService.getEmbeddingProviders() + return res.json(apiResponse) + } catch (error) { + next(error) + } +} + +const getVectorStoreProviders = async (req: Request, res: Response, next: NextFunction) => { + try { + const apiResponse = await documentStoreService.getVectorStoreProviders() + return res.json(apiResponse) + } catch (error) { + next(error) + } +} + +const getRecordManagerProviders = async (req: Request, res: Response, next: NextFunction) => { + try { + const apiResponse = await documentStoreService.getRecordManagerProviders() + return res.json(apiResponse) + } catch (error) { + next(error) + } +} + export default { deleteDocumentStore, createDocumentStore, @@ -260,5 +354,13 @@ export default { previewFileChunks, getDocumentLoaders, deleteDocumentStoreFileChunk, - editDocumentStoreFileChunk + editDocumentStoreFileChunk, + insertIntoVectorStore, + getEmbeddingProviders, + getVectorStoreProviders, + getRecordManagerProviders, + saveVectorStoreConfig, + queryVectorStore, + deleteVectorStoreFromStore, + updateVectorStoreConfigOnly } diff --git a/packages/server/src/database/entities/DocumentStore.ts b/packages/server/src/database/entities/DocumentStore.ts index 3112a62b..694db3e3 100644 --- a/packages/server/src/database/entities/DocumentStore.ts +++ b/packages/server/src/database/entities/DocumentStore.ts @@ -28,4 +28,13 @@ export class DocumentStore implements IDocumentStore { @Column({ nullable: false, type: 'text' }) status: DocumentStoreStatus + + @Column({ nullable: true, type: 'text' }) + vectorStoreConfig: string | null + + @Column({ nullable: true, type: 'text' }) + embeddingConfig: string | null + + @Column({ nullable: true, type: 'text' }) + recordManagerConfig: string | null } diff --git a/packages/server/src/database/migrations/mysql/1715861032479-AddVectorStoreConfigToDocStore.ts b/packages/server/src/database/migrations/mysql/1715861032479-AddVectorStoreConfigToDocStore.ts new file mode 100644 index 00000000..c5d54227 --- /dev/null +++ b/packages/server/src/database/migrations/mysql/1715861032479-AddVectorStoreConfigToDocStore.ts @@ -0,0 +1,18 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class AddVectorStoreConfigToDocStore1715861032479 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + const columnExists = await queryRunner.hasColumn('document_store', 'vectorStoreConfig') + if (!columnExists) { + await queryRunner.query(`ALTER TABLE \`document_store\` ADD COLUMN \`vectorStoreConfig\` TEXT;`) + await queryRunner.query(`ALTER TABLE \`document_store\` ADD COLUMN \`embeddingConfig\` TEXT;`) + await queryRunner.query(`ALTER TABLE \`document_store\` ADD COLUMN \`recordManagerConfig\` TEXT;`) + } + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE \`document_store\` DROP COLUMN \`vectorStoreConfig\`;`) + await queryRunner.query(`ALTER TABLE \`document_store\` DROP COLUMN \`embeddingConfig\`;`) + await queryRunner.query(`ALTER TABLE \`document_store\` DROP COLUMN \`recordManagerConfig\`;`) + } +} diff --git a/packages/server/src/database/migrations/mysql/index.ts b/packages/server/src/database/migrations/mysql/index.ts index 99e98956..fda6caae 100644 --- a/packages/server/src/database/migrations/mysql/index.ts +++ b/packages/server/src/database/migrations/mysql/index.ts @@ -17,6 +17,7 @@ import { AddFeedback1707213626553 } from './1707213626553-AddFeedback' import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHistoryEntity' import { AddLead1710832127079 } from './1710832127079-AddLead' import { AddLeadToChatMessage1711538023578 } from './1711538023578-AddLeadToChatMessage' +import { AddVectorStoreConfigToDocStore1715861032479 } from './1715861032479-AddVectorStoreConfigToDocStore' import { AddDocumentStore1711637331047 } from './1711637331047-AddDocumentStore' import { AddAgentReasoningToChatMessage1714679514451 } from './1714679514451-AddAgentReasoningToChatMessage' import { AddTypeToChatFlow1716300000000 } from './1716300000000-AddTypeToChatFlow' @@ -47,6 +48,7 @@ export const mysqlMigrations = [ AddLeadToChatMessage1711538023578, AddAgentReasoningToChatMessage1714679514451, AddTypeToChatFlow1716300000000, + AddVectorStoreConfigToDocStore1715861032479, AddApiKey1720230151480, AddActionToChatMessage1721078251523, LongTextColumn1722301395521 diff --git a/packages/server/src/database/migrations/postgres/1715861032479-AddVectorStoreConfigToDocStore.ts b/packages/server/src/database/migrations/postgres/1715861032479-AddVectorStoreConfigToDocStore.ts new file mode 100644 index 00000000..c4f884ea --- /dev/null +++ b/packages/server/src/database/migrations/postgres/1715861032479-AddVectorStoreConfigToDocStore.ts @@ -0,0 +1,15 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class AddVectorStoreConfigToDocStore1715861032479 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "document_store" ADD COLUMN IF NOT EXISTS "vectorStoreConfig" TEXT;`) + await queryRunner.query(`ALTER TABLE "document_store" ADD COLUMN IF NOT EXISTS "embeddingConfig" TEXT;`) + await queryRunner.query(`ALTER TABLE "document_store" ADD COLUMN IF NOT EXISTS "recordManagerConfig" TEXT;`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "document_store" DROP COLUMN "vectorStoreConfig";`) + await queryRunner.query(`ALTER TABLE "document_store" DROP COLUMN "embeddingConfig";`) + await queryRunner.query(`ALTER TABLE "document_store" DROP COLUMN "recordManagerConfig";`) + } +} diff --git a/packages/server/src/database/migrations/postgres/index.ts b/packages/server/src/database/migrations/postgres/index.ts index 79760204..e6e154be 100644 --- a/packages/server/src/database/migrations/postgres/index.ts +++ b/packages/server/src/database/migrations/postgres/index.ts @@ -18,6 +18,7 @@ import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHi import { FieldTypes1710497452584 } from './1710497452584-FieldTypes' import { AddLead1710832137905 } from './1710832137905-AddLead' import { AddLeadToChatMessage1711538016098 } from './1711538016098-AddLeadToChatMessage' +import { AddVectorStoreConfigToDocStore1715861032479 } from './1715861032479-AddVectorStoreConfigToDocStore' import { AddDocumentStore1711637331047 } from './1711637331047-AddDocumentStore' import { AddAgentReasoningToChatMessage1714679514451 } from './1714679514451-AddAgentReasoningToChatMessage' import { AddTypeToChatFlow1716300000000 } from './1716300000000-AddTypeToChatFlow' @@ -48,6 +49,7 @@ export const postgresMigrations = [ AddLeadToChatMessage1711538016098, AddAgentReasoningToChatMessage1714679514451, AddTypeToChatFlow1716300000000, + AddVectorStoreConfigToDocStore1715861032479, AddApiKey1720230151480, AddActionToChatMessage1721078251523 ] diff --git a/packages/server/src/database/migrations/sqlite/1715861032479-AddVectorStoreConfigToDocStore.ts b/packages/server/src/database/migrations/sqlite/1715861032479-AddVectorStoreConfigToDocStore.ts new file mode 100644 index 00000000..97e54ac5 --- /dev/null +++ b/packages/server/src/database/migrations/sqlite/1715861032479-AddVectorStoreConfigToDocStore.ts @@ -0,0 +1,15 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class AddVectorStoreConfigToDocStore1715861032479 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "document_store" ADD COLUMN "vectorStoreConfig" TEXT;`) + await queryRunner.query(`ALTER TABLE "document_store" ADD COLUMN "embeddingConfig" TEXT;`) + await queryRunner.query(`ALTER TABLE "document_store" ADD COLUMN "recordManagerConfig" TEXT;`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "document_store" DROP COLUMN "vectorStoreConfig";`) + await queryRunner.query(`ALTER TABLE "document_store" DROP COLUMN "embeddingConfig";`) + await queryRunner.query(`ALTER TABLE "document_store" DROP COLUMN "recordManagerConfig";`) + } +} diff --git a/packages/server/src/database/migrations/sqlite/index.ts b/packages/server/src/database/migrations/sqlite/index.ts index 331cf3d0..69bd8a69 100644 --- a/packages/server/src/database/migrations/sqlite/index.ts +++ b/packages/server/src/database/migrations/sqlite/index.ts @@ -17,6 +17,7 @@ import { AddFeedback1707213619308 } from './1707213619308-AddFeedback' import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHistoryEntity' import { AddLead1710832117612 } from './1710832117612-AddLead' import { AddLeadToChatMessage1711537986113 } from './1711537986113-AddLeadToChatMessage' +import { AddVectorStoreConfigToDocStore1715861032479 } from './1715861032479-AddVectorStoreConfigToDocStore' import { AddDocumentStore1711637331047 } from './1711637331047-AddDocumentStore' import { AddAgentReasoningToChatMessage1714679514451 } from './1714679514451-AddAgentReasoningToChatMessage' import { AddTypeToChatFlow1716300000000 } from './1716300000000-AddTypeToChatFlow' @@ -46,6 +47,7 @@ export const sqliteMigrations = [ AddLeadToChatMessage1711537986113, AddAgentReasoningToChatMessage1714679514451, AddTypeToChatFlow1716300000000, + AddVectorStoreConfigToDocStore1715861032479, AddApiKey1720230151480, AddActionToChatMessage1721078251523 ] diff --git a/packages/server/src/routes/documentstore/index.ts b/packages/server/src/routes/documentstore/index.ts index 19cb0c0b..987136fc 100644 --- a/packages/server/src/routes/documentstore/index.ts +++ b/packages/server/src/routes/documentstore/index.ts @@ -16,7 +16,7 @@ router.delete('/store/:id', documentStoreController.deleteDocumentStore) /** Component Nodes = Document Store - Loaders */ // Get all loaders -router.get('/loaders', documentStoreController.getDocumentLoaders) +router.get('/components/loaders', documentStoreController.getDocumentLoaders) // delete loader from document store router.delete('/loader/:id/:loaderId', documentStoreController.deleteLoaderFromDocumentStore) @@ -33,4 +33,22 @@ router.put('/chunks/:storeId/:loaderId/:chunkId', documentStoreController.editDo // Get all file chunks from the store router.get('/chunks/:storeId/:fileId/:pageNo', documentStoreController.getDocumentStoreFileChunks) +// add chunks to the selected vector store +router.post('/vectorstore/insert', documentStoreController.insertIntoVectorStore) +// save the selected vector store +router.post('/vectorstore/save', documentStoreController.saveVectorStoreConfig) +// delete data from the selected vector store +router.delete('/vectorstore/:storeId', documentStoreController.deleteVectorStoreFromStore) +// query the vector store +router.post('/vectorstore/query', documentStoreController.queryVectorStore) +// Get all embedding providers +router.get('/components/embeddings', documentStoreController.getEmbeddingProviders) +// Get all vector store providers +router.get('/components/vectorstore', documentStoreController.getVectorStoreProviders) +// Get all Record Manager providers +router.get('/components/recordmanager', documentStoreController.getRecordManagerProviders) + +// update the selected vector store from the playground +router.post('/vectorstore/update', documentStoreController.updateVectorStoreConfigOnly) + export default router diff --git a/packages/server/src/services/documentstore/index.ts b/packages/server/src/services/documentstore/index.ts index ff758642..05a69342 100644 --- a/packages/server/src/services/documentstore/index.ts +++ b/packages/server/src/services/documentstore/index.ts @@ -1,6 +1,5 @@ import { getRunningExpressApp } from '../../utils/getRunningExpressApp' import { DocumentStore } from '../../database/entities/DocumentStore' -// @ts-ignore import { addSingleFileToStorage, getFileFromStorage, @@ -10,22 +9,29 @@ import { removeSpecificFileFromStorage } from 'flowise-components' import { + chatType, DocumentStoreStatus, IDocumentStoreFileChunkPagedResponse, IDocumentStoreLoader, IDocumentStoreLoaderFile, IDocumentStoreLoaderForPreview, - IDocumentStoreWhereUsed + IDocumentStoreWhereUsed, + INodeData } from '../../Interface' import { DocumentStoreFileChunk } from '../../database/entities/DocumentStoreFileChunk' import { v4 as uuidv4 } from 'uuid' -import { databaseEntities } from '../../utils' +import { databaseEntities, getAppVersion, saveUpsertFlowData } from '../../utils' import logger from '../../utils/logger' import nodesService from '../nodes' import { InternalFlowiseError } from '../../errors/internalFlowiseError' import { StatusCodes } from 'http-status-codes' import { getErrorMessage } from '../../errors/utils' import { ChatFlow } from '../../database/entities/ChatFlow' +import { Document } from '@langchain/core/documents' +import { App } from '../../index' +import { UpsertHistory } from '../../database/entities/UpsertHistory' +import { cloneDeep, omit } from 'lodash' +import telemetryService from '../telemetry' const DOCUMENT_STORE_BASE_FOLDER = 'docustore' @@ -234,8 +240,16 @@ const deleteDocumentStore = async (storeId: string) => { const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({ id: storeId }) - if (!entity) throw new Error(`Document store ${storeId} not found`) + if (!entity) { + throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document store ${storeId} not found`) + } await removeFilesFromStorage(DOCUMENT_STORE_BASE_FOLDER, entity.id) + + // delete upsert history + await appServer.AppDataSource.getRepository(UpsertHistory).delete({ + chatflowid: storeId + }) + // now delete the store const tbd = await appServer.AppDataSource.getRepository(DocumentStore).delete({ id: storeId @@ -285,6 +299,83 @@ const deleteDocumentStoreFileChunk = async (storeId: string, docId: string, chun } } +const deleteVectorStoreFromStore = async (storeId: string) => { + try { + const appServer = getRunningExpressApp() + const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({ + id: storeId + }) + if (!entity) { + throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document store ${storeId} not found`) + } + + if (!entity.embeddingConfig) { + throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Embedding for Document store ${storeId} not found`) + } + + if (!entity.vectorStoreConfig) { + throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Vector Store for Document store ${storeId} not found`) + } + + if (!entity.recordManagerConfig) { + throw new InternalFlowiseError( + StatusCodes.NOT_FOUND, + `Record Manager for Document Store ${storeId} is needed to delete data from Vector Store` + ) + } + + const options: ICommonObject = { + chatflowid: storeId, + appDataSource: appServer.AppDataSource, + databaseEntities, + logger + } + + // Get Record Manager Instance + const recordManagerConfig = JSON.parse(entity.recordManagerConfig) + const recordManagerObj = await _createRecordManagerObject( + appServer, + { recordManagerName: recordManagerConfig.name, recordManagerConfig: recordManagerConfig.config }, + options + ) + + // Get Embeddings Instance + const embeddingConfig = JSON.parse(entity.embeddingConfig) + const embeddingObj = await _createEmbeddingsObject( + appServer, + { embeddingName: embeddingConfig.name, embeddingConfig: embeddingConfig.config }, + options + ) + + // Get Vector Store Node Data + const vectorStoreConfig = JSON.parse(entity.vectorStoreConfig) + const vStoreNodeData = _createVectorStoreNodeData( + appServer, + { vectorStoreName: vectorStoreConfig.name, vectorStoreConfig: vectorStoreConfig.config }, + embeddingObj, + recordManagerObj + ) + + // Get Vector Store Instance + const vectorStoreObj = await _createVectorStoreObject( + appServer, + { vectorStoreName: vectorStoreConfig.name, vectorStoreConfig: vectorStoreConfig.config }, + vStoreNodeData + ) + const idsToDelete: string[] = [] // empty ids because we get it dynamically from the record manager + + // Call the delete method of the vector store + if (vectorStoreObj.vectorStoreMethods.delete) { + await vectorStoreObj.vectorStoreMethods.delete(vStoreNodeData, idsToDelete, options) + } + } catch (error) { + throw new InternalFlowiseError( + StatusCodes.INTERNAL_SERVER_ERROR, + `Error: documentStoreServices.deleteVectorStoreFromStore - ${getErrorMessage(error)}` + ) + } +} + const editDocumentStoreFileChunk = async (storeId: string, docId: string, chunkId: string, content: string, metadata: ICommonObject) => { try { const appServer = getRunningExpressApp() @@ -700,6 +791,417 @@ const updateDocumentStoreUsage = async (chatId: string, storeId: string | undefi } } +const updateVectorStoreConfigOnly = async (data: ICommonObject) => { + try { + const appServer = getRunningExpressApp() + const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({ + id: data.storeId + }) + if (!entity) { + throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document store ${data.storeId} not found`) + } + + if (data.vectorStoreName) { + entity.vectorStoreConfig = JSON.stringify({ + config: data.vectorStoreConfig, + name: data.vectorStoreName + }) + + const updatedEntity = await appServer.AppDataSource.getRepository(DocumentStore).save(entity) + return updatedEntity + } + return {} + } catch (error) { + throw new InternalFlowiseError( + StatusCodes.INTERNAL_SERVER_ERROR, + `Error: documentStoreServices.updateVectorStoreConfig - ${getErrorMessage(error)}` + ) + } +} +const saveVectorStoreConfig = async (data: ICommonObject) => { + try { + const appServer = getRunningExpressApp() + const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({ + id: data.storeId + }) + if (!entity) { + throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document store ${data.storeId} not found`) + } + + if (data.embeddingName) { + entity.embeddingConfig = JSON.stringify({ + config: data.embeddingConfig, + name: data.embeddingName + }) + } else if (!data.embeddingName && !data.embeddingConfig) { + entity.embeddingConfig = null + } + + if (data.vectorStoreName) { + entity.vectorStoreConfig = JSON.stringify({ + config: data.vectorStoreConfig, + name: data.vectorStoreName + }) + } else if (!data.vectorStoreName && !data.vectorStoreConfig) { + entity.vectorStoreConfig = null + } + + if (data.recordManagerName) { + entity.recordManagerConfig = JSON.stringify({ + config: data.recordManagerConfig, + name: data.recordManagerName + }) + } else if (!data.recordManagerName && !data.recordManagerConfig) { + entity.recordManagerConfig = null + } + + if (entity.status !== DocumentStoreStatus.UPSERTED && (data.vectorStoreName || data.recordManagerName || data.embeddingName)) { + // if the store is not already in sync, mark it as sync + // this also means that the store is not yet sync'ed to vector store + entity.status = DocumentStoreStatus.SYNC + } + await appServer.AppDataSource.getRepository(DocumentStore).save(entity) + return entity + } catch (error) { + throw new InternalFlowiseError( + StatusCodes.INTERNAL_SERVER_ERROR, + `Error: documentStoreServices.saveVectorStoreConfig - ${getErrorMessage(error)}` + ) + } +} + +const insertIntoVectorStore = async (data: ICommonObject) => { + try { + const appServer = getRunningExpressApp() + const entity = await saveVectorStoreConfig(data) + entity.status = DocumentStoreStatus.UPSERTING + await appServer.AppDataSource.getRepository(DocumentStore).save(entity) + + // TODO: to be moved into a worker thread... + const indexResult = await _insertIntoVectorStoreWorkerThread(data) + return indexResult + } catch (error) { + throw new InternalFlowiseError( + StatusCodes.INTERNAL_SERVER_ERROR, + `Error: documentStoreServices.insertIntoVectorStore - ${getErrorMessage(error)}` + ) + } +} + +const _insertIntoVectorStoreWorkerThread = async (data: ICommonObject) => { + try { + const appServer = getRunningExpressApp() + const entity = await saveVectorStoreConfig(data) + let upsertHistory: Record = {} + const chatflowid = data.storeId // fake chatflowid because this is not tied to any chatflow + + const options: ICommonObject = { + chatflowid, + appDataSource: appServer.AppDataSource, + databaseEntities, + logger + } + + let recordManagerObj = undefined + + // Get Record Manager Instance + if (data.recordManagerName && data.recordManagerConfig) { + recordManagerObj = await _createRecordManagerObject(appServer, data, options, upsertHistory) + } + + // Get Embeddings Instance + const embeddingObj = await _createEmbeddingsObject(appServer, data, options, upsertHistory) + + // Get Vector Store Node Data + const vStoreNodeData = _createVectorStoreNodeData(appServer, data, embeddingObj, recordManagerObj) + // Prepare docs for upserting + const chunks = await appServer.AppDataSource.getRepository(DocumentStoreFileChunk).find({ + where: { + storeId: data.storeId + } + }) + const docs: Document[] = chunks.map((chunk: DocumentStoreFileChunk) => { + return new Document({ + pageContent: chunk.pageContent, + metadata: JSON.parse(chunk.metadata) + }) + }) + vStoreNodeData.inputs.document = docs + + // Get Vector Store Instance + const vectorStoreObj = await _createVectorStoreObject(appServer, data, vStoreNodeData, upsertHistory) + const indexResult = await vectorStoreObj.vectorStoreMethods.upsert(vStoreNodeData, options) + + // Save to DB + if (indexResult) { + const result = cloneDeep(upsertHistory) + result['flowData'] = JSON.stringify(result['flowData']) + result['result'] = JSON.stringify(omit(indexResult, ['totalKeys', 'addedDocs'])) + result.chatflowid = chatflowid + const newUpsertHistory = new UpsertHistory() + Object.assign(newUpsertHistory, result) + const upsertHistoryItem = appServer.AppDataSource.getRepository(UpsertHistory).create(newUpsertHistory) + await appServer.AppDataSource.getRepository(UpsertHistory).save(upsertHistoryItem) + } + + await telemetryService.createEvent({ + name: `vector_upserted`, + data: { + version: await getAppVersion(), + chatlowId: chatflowid, + type: chatType.INTERNAL, + flowGraph: omit(indexResult['result'], ['totalKeys', 'addedDocs']) + } + }) + + entity.status = DocumentStoreStatus.UPSERTED + await appServer.AppDataSource.getRepository(DocumentStore).save(entity) + + return indexResult ?? { result: 'Successfully Upserted' } + } catch (error) { + throw new InternalFlowiseError( + StatusCodes.INTERNAL_SERVER_ERROR, + `Error: documentStoreServices._insertIntoVectorStoreWorkerThread - ${getErrorMessage(error)}` + ) + } +} + +// Get all component nodes - Embeddings +const getEmbeddingProviders = async () => { + try { + const dbResponse = await nodesService.getAllNodesForCategory('Embeddings') + return dbResponse.filter((node) => !node.tags?.includes('LlamaIndex')) + } catch (error) { + throw new InternalFlowiseError( + StatusCodes.INTERNAL_SERVER_ERROR, + `Error: documentStoreServices.getEmbeddingProviders - ${getErrorMessage(error)}` + ) + } +} + +// Get all component nodes - Vector Stores +const getVectorStoreProviders = async () => { + try { + const dbResponse = await nodesService.getAllNodesForCategory('Vector Stores') + return dbResponse.filter((node) => !node.tags?.includes('LlamaIndex') && node.name !== 'documentStoreVS') + } catch (error) { + throw new InternalFlowiseError( + StatusCodes.INTERNAL_SERVER_ERROR, + `Error: documentStoreServices.getVectorStoreProviders - ${getErrorMessage(error)}` + ) + } +} +// Get all component nodes - Vector Stores +const getRecordManagerProviders = async () => { + try { + const dbResponse = await nodesService.getAllNodesForCategory('Record Manager') + return dbResponse.filter((node) => !node.tags?.includes('LlamaIndex')) + } catch (error) { + throw new InternalFlowiseError( + StatusCodes.INTERNAL_SERVER_ERROR, + `Error: documentStoreServices.getRecordManagerProviders - ${getErrorMessage(error)}` + ) + } +} + +const queryVectorStore = async (data: ICommonObject) => { + try { + const appServer = getRunningExpressApp() + const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({ + id: data.storeId + }) + if (!entity) { + throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Document store ${data.storeId} not found`) + } + const options: ICommonObject = { + chatflowid: uuidv4(), + appDataSource: appServer.AppDataSource, + databaseEntities, + logger + } + + if (!entity.embeddingConfig) { + throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Embedding for ${data.storeId} is not configured`) + } + + if (!entity.vectorStoreConfig) { + throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Vector Store for ${data.storeId} is not configured`) + } + + const embeddingConfig = JSON.parse(entity.embeddingConfig) + data.embeddingName = embeddingConfig.name + data.embeddingConfig = embeddingConfig.config + let embeddingObj = await _createEmbeddingsObject(appServer, data, options) + + const vsConfig = JSON.parse(entity.vectorStoreConfig) + data.vectorStoreName = vsConfig.name + data.vectorStoreConfig = vsConfig.config + if (data.inputs) { + data.vectorStoreConfig = { ...vsConfig.config, ...data.inputs } + } + + const vStoreNodeData = _createVectorStoreNodeData(appServer, data, embeddingObj, undefined) + + // Get Vector Store Instance + const vectorStoreObj = await _createVectorStoreObject(appServer, data, vStoreNodeData) + const retriever = await vectorStoreObj.init(vStoreNodeData, '', options) + if (!retriever) { + throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Failed to create retriever`) + } + const startMillis = Date.now() + const results = await retriever.invoke(data.query, undefined) + if (!results) { + throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Failed to retrieve results`) + } + const endMillis = Date.now() + const timeTaken = endMillis - startMillis + const docs: any = results.map((result: IDocument) => { + return { + pageContent: result.pageContent, + metadata: result.metadata, + id: uuidv4() + } + }) + // query our document store chunk with the storeId and pageContent + for (const doc of docs) { + const documentStoreChunk = await appServer.AppDataSource.getRepository(DocumentStoreFileChunk).findOneBy({ + storeId: data.storeId, + pageContent: doc.pageContent + }) + if (documentStoreChunk) { + doc.id = documentStoreChunk.id + doc.chunkNo = documentStoreChunk.chunkNo + } else { + // this should not happen, only possible if the vector store has more content + // than our document store + doc.id = uuidv4() + doc.chunkNo = -1 + } + } + + return { + timeTaken: timeTaken, + docs: docs + } + } catch (error) { + throw new InternalFlowiseError( + StatusCodes.INTERNAL_SERVER_ERROR, + `Error: documentStoreServices.queryVectorStore - ${getErrorMessage(error)}` + ) + } +} + +const _createEmbeddingsObject = async ( + appServer: App, + data: ICommonObject, + options: ICommonObject, + upsertHistory?: Record +): Promise => { + // prepare embedding node data + const embeddingComponent = appServer.nodesPool.componentNodes[data.embeddingName] + const embeddingNodeData: any = { + inputs: { ...data.embeddingConfig }, + outputs: { output: 'document' }, + id: `${embeddingComponent.name}_0`, + label: embeddingComponent.label, + name: embeddingComponent.name, + category: embeddingComponent.category, + inputParams: embeddingComponent.inputs || [] + } + if (data.embeddingConfig.credential) { + embeddingNodeData.credential = data.embeddingConfig.credential + } + + // save to upsert history + if (upsertHistory) upsertHistory['flowData'] = saveUpsertFlowData(embeddingNodeData, upsertHistory) + + // init embedding object + const embeddingNodeInstanceFilePath = embeddingComponent.filePath as string + const embeddingNodeModule = await import(embeddingNodeInstanceFilePath) + const embeddingNodeInstance = new embeddingNodeModule.nodeClass() + const embeddingObj = await embeddingNodeInstance.init(embeddingNodeData, '', options) + if (!embeddingObj) { + throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Failed to create EmbeddingObj`) + } + return embeddingObj +} + +const _createRecordManagerObject = async ( + appServer: App, + data: ICommonObject, + options: ICommonObject, + upsertHistory?: Record +) => { + // prepare record manager node data + const recordManagerComponent = appServer.nodesPool.componentNodes[data.recordManagerName] + const rmNodeData: any = { + inputs: { ...data.recordManagerConfig }, + id: `${recordManagerComponent.name}_0`, + inputParams: recordManagerComponent.inputs, + label: recordManagerComponent.label, + name: recordManagerComponent.name, + category: recordManagerComponent.category + } + if (data.recordManagerConfig.credential) { + rmNodeData.credential = data.recordManagerConfig.credential + } + + // save to upsert history + if (upsertHistory) upsertHistory['flowData'] = saveUpsertFlowData(rmNodeData, upsertHistory) + + // init record manager object + const rmNodeInstanceFilePath = recordManagerComponent.filePath as string + const rmNodeModule = await import(rmNodeInstanceFilePath) + const rmNodeInstance = new rmNodeModule.nodeClass() + const recordManagerObj = await rmNodeInstance.init(rmNodeData, '', options) + if (!recordManagerObj) { + throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Failed to create RecordManager obj`) + } + return recordManagerObj +} + +const _createVectorStoreNodeData = (appServer: App, data: ICommonObject, embeddingObj: any, recordManagerObj?: any) => { + const vectorStoreComponent = appServer.nodesPool.componentNodes[data.vectorStoreName] + const vStoreNodeData: any = { + id: `${vectorStoreComponent.name}_0`, + inputs: { ...data.vectorStoreConfig }, + outputs: { output: 'retriever' }, + label: vectorStoreComponent.label, + name: vectorStoreComponent.name, + category: vectorStoreComponent.category + } + if (data.vectorStoreConfig.credential) { + vStoreNodeData.credential = data.vectorStoreConfig.credential + } + + if (embeddingObj) { + vStoreNodeData.inputs.embeddings = embeddingObj + } + + if (recordManagerObj) { + vStoreNodeData.inputs.recordManager = recordManagerObj + } + + // Get all input params except the ones that are anchor points to avoid JSON stringify circular error + const filterInputParams = ['document', 'embeddings', 'recordManager'] + const inputParams = vectorStoreComponent.inputs?.filter((input) => !filterInputParams.includes(input.name)) + vStoreNodeData.inputParams = inputParams + return vStoreNodeData +} + +const _createVectorStoreObject = async ( + appServer: App, + data: ICommonObject, + vStoreNodeData: INodeData, + upsertHistory?: Record +) => { + const vStoreNodeInstanceFilePath = appServer.nodesPool.componentNodes[data.vectorStoreName].filePath as string + const vStoreNodeModule = await import(vStoreNodeInstanceFilePath) + const vStoreNodeInstance = new vStoreNodeModule.nodeClass() + if (upsertHistory) upsertHistory['flowData'] = saveUpsertFlowData(vStoreNodeData, upsertHistory) + return vStoreNodeInstance +} + export default { updateDocumentStoreUsage, deleteDocumentStore, @@ -714,5 +1216,13 @@ export default { processAndSaveChunks, deleteDocumentStoreFileChunk, editDocumentStoreFileChunk, - getDocumentLoaders + getDocumentLoaders, + insertIntoVectorStore, + getEmbeddingProviders, + getVectorStoreProviders, + getRecordManagerProviders, + saveVectorStoreConfig, + queryVectorStore, + deleteVectorStoreFromStore, + updateVectorStoreConfigOnly } diff --git a/packages/server/src/utils/index.ts b/packages/server/src/utils/index.ts index 6d18181e..6fe07ffa 100644 --- a/packages/server/src/utils/index.ts +++ b/packages/server/src/utils/index.ts @@ -547,7 +547,8 @@ export const buildFlow = async ({ uploads, baseURL, socketIO, - socketIOClientId + socketIOClientId, + componentNodes: componentNodes as ICommonObject }) // Save dynamic variables diff --git a/packages/ui/src/api/documentstore.js b/packages/ui/src/api/documentstore.js index cf6d1ae2..0f96b66f 100644 --- a/packages/ui/src/api/documentstore.js +++ b/packages/ui/src/api/documentstore.js @@ -1,7 +1,7 @@ import client from './client' const getAllDocumentStores = () => client.get('/document-store/stores') -const getDocumentLoaders = () => client.get('/document-store/loaders') +const getDocumentLoaders = () => client.get('/document-store/components/loaders') const getSpecificDocumentStore = (id) => client.get(`/document-store/store/${id}`) const createDocumentStore = (body) => client.post(`/document-store/store`, body) const updateDocumentStore = (id, body) => client.put(`/document-store/store/${id}`, body) @@ -16,6 +16,15 @@ const getFileChunks = (storeId, fileId, pageNo) => client.get(`/document-store/c const previewChunks = (body) => client.post('/document-store/loader/preview', body) const processChunks = (body) => client.post(`/document-store/loader/process`, body) +const insertIntoVectorStore = (body) => client.post(`/document-store/vectorstore/insert`, body) +const saveVectorStoreConfig = (body) => client.post(`/document-store/vectorstore/save`, body) +const updateVectorStoreConfig = (body) => client.post(`/document-store/vectorstore/update`, body) +const deleteVectorStoreDataFromStore = (storeId) => client.delete(`/document-store/vectorstore/${storeId}`) +const queryVectorStore = (body) => client.post(`/document-store/vectorstore/query`, body) +const getVectorStoreProviders = () => client.get('/document-store/components/vectorstore') +const getEmbeddingProviders = () => client.get('/document-store/components/embeddings') +const getRecordManagerProviders = () => client.get('/document-store/components/recordmanager') + export default { getAllDocumentStores, getSpecificDocumentStore, @@ -28,5 +37,13 @@ export default { getDocumentLoaders, deleteChunkFromStore, editChunkFromStore, - deleteDocumentStore + deleteDocumentStore, + insertIntoVectorStore, + getVectorStoreProviders, + getEmbeddingProviders, + getRecordManagerProviders, + saveVectorStoreConfig, + queryVectorStore, + deleteVectorStoreDataFromStore, + updateVectorStoreConfig } diff --git a/packages/ui/src/routes/MainRoutes.jsx b/packages/ui/src/routes/MainRoutes.jsx index 76783dc0..53afab08 100644 --- a/packages/ui/src/routes/MainRoutes.jsx +++ b/packages/ui/src/routes/MainRoutes.jsx @@ -33,6 +33,8 @@ const Documents = Loadable(lazy(() => import('@/views/docstore'))) const DocumentStoreDetail = Loadable(lazy(() => import('@/views/docstore/DocumentStoreDetail'))) const ShowStoredChunks = Loadable(lazy(() => import('@/views/docstore/ShowStoredChunks'))) const LoaderConfigPreviewChunks = Loadable(lazy(() => import('@/views/docstore/LoaderConfigPreviewChunks'))) +const VectorStoreConfigure = Loadable(lazy(() => import('@/views/docstore/VectorStoreConfigure'))) +const VectorStoreQuery = Loadable(lazy(() => import('@/views/docstore/VectorStoreQuery'))) // ==============================|| MAIN ROUTING ||============================== // @@ -91,6 +93,14 @@ const MainRoutes = { { path: '/document-stores/:id/:name', element: + }, + { + path: '/document-stores/vector/:id', + element: + }, + { + path: '/document-stores/query/:id', + element: } ] } diff --git a/packages/ui/src/views/docstore/ComponentsListDialog.jsx b/packages/ui/src/views/docstore/ComponentsListDialog.jsx new file mode 100644 index 00000000..9e4658f3 --- /dev/null +++ b/packages/ui/src/views/docstore/ComponentsListDialog.jsx @@ -0,0 +1,190 @@ +import { useState, useEffect } from 'react' +import { createPortal } from 'react-dom' +import { useSelector, useDispatch } from 'react-redux' +import PropTypes from 'prop-types' +import { List, ListItemButton, Dialog, DialogContent, DialogTitle, Box, OutlinedInput, InputAdornment, Typography } from '@mui/material' +import { useTheme } from '@mui/material/styles' +import { IconSearch, IconX } from '@tabler/icons-react' + +// API + +// const +import { baseURL } from '@/store/constant' +import { HIDE_CANVAS_DIALOG, SHOW_CANVAS_DIALOG } from '@/store/actions' +import useApi from '@/hooks/useApi' + +const ComponentsListDialog = ({ show, dialogProps, onCancel, apiCall, onSelected }) => { + const portalElement = document.getElementById('portal') + const customization = useSelector((state) => state.customization) + const dispatch = useDispatch() + const theme = useTheme() + const [searchValue, setSearchValue] = useState('') + const [provider, setProvider] = useState([]) + + const getProvidersApi = useApi(apiCall) + + const onSearchChange = (val) => { + setSearchValue(val) + } + + function filterFlows(data) { + return data?.name?.toLowerCase().indexOf(searchValue.toLowerCase()) > -1 + } + + useEffect(() => { + // if (dialogProps.embeddingsProvider) { + // setProvider(dialogProps.provider) + // } + }, [dialogProps]) + + useEffect(() => { + getProvidersApi.request() + + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []) + + useEffect(() => { + if (getProvidersApi.data) { + setProvider(getProvidersApi.data) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [getProvidersApi.data]) + + useEffect(() => { + if (show) dispatch({ type: SHOW_CANVAS_DIALOG }) + else dispatch({ type: HIDE_CANVAS_DIALOG }) + return () => dispatch({ type: HIDE_CANVAS_DIALOG }) + }, [show, dispatch]) + + const component = show ? ( + + + {dialogProps.title} + + + + onSearchChange(e.target.value)} + placeholder='Search' + startAdornment={ + + + + } + endAdornment={ + + onSearchChange('')} + style={{ + cursor: 'pointer' + }} + /> + + } + aria-describedby='search-helper-text' + inputProps={{ + 'aria-label': 'weight' + }} + /> + + + {[...provider].filter(filterFlows).map((loader) => ( + onSelected(loader)} + sx={{ + border: 1, + borderColor: theme.palette.grey[900] + 25, + borderRadius: 2, + display: 'flex', + alignItems: 'center', + justifyContent: 'start', + textAlign: 'left', + gap: 1, + p: 2 + }} + > +
+ {loader.name} +
+ {loader.label} + + ))} + + + + ) : null + + return createPortal(component, portalElement) +} + +ComponentsListDialog.propTypes = { + show: PropTypes.bool, + dialogProps: PropTypes.object, + onCancel: PropTypes.func, + apiCall: PropTypes.func, + onSelected: PropTypes.func +} + +export default ComponentsListDialog diff --git a/packages/ui/src/views/docstore/DeleteDocStoreDialog.jsx b/packages/ui/src/views/docstore/DeleteDocStoreDialog.jsx new file mode 100644 index 00000000..d1e67c94 --- /dev/null +++ b/packages/ui/src/views/docstore/DeleteDocStoreDialog.jsx @@ -0,0 +1,246 @@ +import { useState, useEffect } from 'react' +import { createPortal } from 'react-dom' +import PropTypes from 'prop-types' +import { cloneDeep } from 'lodash' +import { + Button, + Box, + Paper, + Accordion, + AccordionSummary, + AccordionDetails, + Dialog, + DialogContent, + DialogTitle, + Typography, + Table, + TableBody, + TableContainer, + TableRow, + TableCell, + Checkbox, + FormControlLabel, + DialogActions +} from '@mui/material' +import ExpandMoreIcon from '@mui/icons-material/ExpandMore' +import { TableViewOnly } from '@/ui-component/table/Table' +import { v4 as uuidv4 } from 'uuid' + +// const +import { baseURL } from '@/store/constant' +import nodesApi from '@/api/nodes' + +// Hooks +import useApi from '@/hooks/useApi' +import { initNode } from '@/utils/genericHelper' + +const DeleteDocStoreDialog = ({ show, dialogProps, onCancel, onDelete }) => { + const portalElement = document.getElementById('portal') + const [nodeConfigExpanded, setNodeConfigExpanded] = useState({}) + const [removeFromVS, setRemoveFromVS] = useState(false) + const [vsFlowData, setVSFlowData] = useState([]) + const [rmFlowData, setRMFlowData] = useState([]) + + const getSpecificNodeApi = useApi(nodesApi.getSpecificNode) + + const handleAccordionChange = (nodeName) => (event, isExpanded) => { + const accordianNodes = { ...nodeConfigExpanded } + accordianNodes[nodeName] = isExpanded + setNodeConfigExpanded(accordianNodes) + } + + useEffect(() => { + if (dialogProps.recordManagerConfig) { + const nodeName = dialogProps.recordManagerConfig.name + if (nodeName) getSpecificNodeApi.request(nodeName) + + if (dialogProps.vectorStoreConfig) { + const nodeName = dialogProps.vectorStoreConfig.name + if (nodeName) getSpecificNodeApi.request(nodeName) + } + } + + return () => { + setNodeConfigExpanded({}) + setRemoveFromVS(false) + setVSFlowData([]) + setRMFlowData([]) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [dialogProps]) + + useEffect(() => { + if (getSpecificNodeApi.data) { + const nodeData = cloneDeep(initNode(getSpecificNodeApi.data, uuidv4())) + + let config = 'vectorStoreConfig' + if (nodeData.category === 'Record Manager') config = 'recordManagerConfig' + + const paramValues = [] + + for (const inputName in dialogProps[config].config) { + const inputParam = nodeData.inputParams.find((inp) => inp.name === inputName) + + if (!inputParam) continue + + if (inputParam.type === 'credential') continue + + let paramValue = {} + + const inputValue = dialogProps[config].config[inputName] + + if (!inputValue) continue + + if (typeof inputValue === 'string' && inputValue.startsWith('{{') && inputValue.endsWith('}}')) { + continue + } + + paramValue = { + label: inputParam?.label, + name: inputParam?.name, + type: inputParam?.type, + value: inputValue + } + paramValues.push(paramValue) + } + + if (config === 'vectorStoreConfig') { + setVSFlowData([ + { + label: nodeData.label, + name: nodeData.name, + category: nodeData.category, + id: nodeData.id, + paramValues + } + ]) + } else if (config === 'recordManagerConfig') { + setRMFlowData([ + { + label: nodeData.label, + name: nodeData.name, + category: nodeData.category, + id: nodeData.id, + paramValues + } + ]) + } + } + + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [getSpecificNodeApi.data]) + + const component = show ? ( + + + {dialogProps.title} + + + {dialogProps.description} + {dialogProps.type === 'STORE' && dialogProps.recordManagerConfig && ( + setRemoveFromVS(event.target.checked)} />} + label='Remove data from vector store' + /> + )} + {removeFromVS && ( +
+ + + + + + + {([...vsFlowData, ...rmFlowData] || []).map((node, index) => { + return ( + + } + aria-controls={`nodes-accordian-${node.name}`} + id={`nodes-accordian-header-${node.name}`} + > +
+
+ {node.name} +
+ {node.label} +
+ + + {node.paramValues[0] && ( + + )} + + + ) + })} + + + + +
+ + + * Only data that were upserted with Record Manager will be deleted from vector store + +
+ )} + + + + + + + ) : null + + return createPortal(component, portalElement) +} + +DeleteDocStoreDialog.propTypes = { + show: PropTypes.bool, + dialogProps: PropTypes.object, + onCancel: PropTypes.func, + onDelete: PropTypes.func +} + +export default DeleteDocStoreDialog diff --git a/packages/ui/src/views/docstore/DocStoreInputHandler.jsx b/packages/ui/src/views/docstore/DocStoreInputHandler.jsx index 2de05a58..cb4e75b1 100644 --- a/packages/ui/src/views/docstore/DocStoreInputHandler.jsx +++ b/packages/ui/src/views/docstore/DocStoreInputHandler.jsx @@ -118,8 +118,9 @@ const DocStoreInputHandler = ({ inputParam, data, disabled = false }) => { )} {inputParam.type === 'credential' && ( { data.credential = newValue @@ -189,6 +190,7 @@ const DocStoreInputHandler = ({ inputParam, data, disabled = false }) => { )} {inputParam.type === 'options' && ( { )} {inputParam.type === 'multiOptions' && ( { )} {inputParam.type === 'asyncOptions' && ( <> - {data.inputParams.length === 1 &&
} + {data.inputParams?.length === 1 &&
}
{ const enqueueSnackbar = (...args) => dispatch(enqueueSnackbarAction(...args)) const closeSnackbar = (...args) => dispatch(closeSnackbarAction(...args)) - const { confirm } = useConfirm() const getSpecificDocumentStore = useApi(documentsApi.getSpecificDocumentStore) const [error, setError] = useState(null) const [isLoading, setLoading] = useState(true) + const [isBackdropLoading, setBackdropLoading] = useState(false) const [showDialog, setShowDialog] = useState(false) const [documentStore, setDocumentStore] = useState({}) const [dialogProps, setDialogProps] = useState({}) const [showDocumentLoaderListDialog, setShowDocumentLoaderListDialog] = useState(false) const [documentLoaderListDialogProps, setDocumentLoaderListDialogProps] = useState({}) + const [showDeleteDocStoreDialog, setShowDeleteDocStoreDialog] = useState(false) + const [deleteDocStoreDialogProps, setDeleteDocStoreDialogProps] = useState({}) const URLpath = document.location.pathname.toString().split('/') const storeId = URLpath[URLpath.length - 1] === 'document-stores' ? '' : URLpath[URLpath.length - 1] @@ -144,11 +155,19 @@ const DocumentStoreDetails = () => { navigate('/document-stores/chunks/' + storeId + '/' + id) } + const showVectorStoreQuery = (id) => { + navigate('/document-stores/query/' + id) + } + const onDocLoaderSelected = (docLoaderComponentName) => { setShowDocumentLoaderListDialog(false) navigate('/document-stores/' + storeId + '/' + docLoaderComponentName) } + const showVectorStore = (id) => { + navigate('/document-stores/vector/' + id) + } + const listLoaders = () => { const dialogProp = { title: 'Select Document Loader' @@ -157,18 +176,60 @@ const DocumentStoreDetails = () => { setShowDocumentLoaderListDialog(true) } - const onLoaderDelete = async (file) => { - const confirmPayload = { - title: `Delete`, - description: `Delete Loader ${file.loaderName} ? This will delete all the associated document chunks.`, - confirmButtonName: 'Delete', - cancelButtonName: 'Cancel' + const deleteVectorStoreDataFromStore = async (storeId) => { + try { + await documentsApi.deleteVectorStoreDataFromStore(storeId) + } catch (error) { + console.error(error) } - const isConfirmed = await confirm(confirmPayload) + } - if (isConfirmed) { + const onDocStoreDelete = async (type, removeFromVectorStore) => { + setBackdropLoading(true) + if (type === 'STORE') { + if (removeFromVectorStore) { + await deleteVectorStoreDataFromStore(storeId) + } + try { + const deleteResp = await documentsApi.deleteDocumentStore(storeId) + setBackdropLoading(false) + if (deleteResp.data) { + enqueueSnackbar({ + message: 'Store, Loader and associated document chunks deleted', + options: { + key: new Date().getTime() + Math.random(), + variant: 'success', + action: (key) => ( + + ) + } + }) + navigate('/document-stores/') + } + } catch (error) { + setBackdropLoading(false) + setError(error) + const errorData = error.response.data || `${error.response.status}: ${error.response.statusText}` + enqueueSnackbar({ + message: `Failed to delete loader: ${errorData}`, + options: { + key: new Date().getTime() + Math.random(), + variant: 'error', + persist: true, + action: (key) => ( + + ) + } + }) + } + } else if (type === 'LOADER') { try { const deleteResp = await documentsApi.deleteLoaderFromStore(storeId, file.id) + setBackdropLoading(false) if (deleteResp.data) { enqueueSnackbar({ message: 'Loader and associated document chunks deleted', @@ -186,6 +247,7 @@ const DocumentStoreDetails = () => { } } catch (error) { setError(error) + setBackdropLoading(false) const errorData = error.response.data || `${error.response.status}: ${error.response.statusText}` enqueueSnackbar({ message: `Failed to delete loader: ${errorData}`, @@ -204,51 +266,30 @@ const DocumentStoreDetails = () => { } } - const onStoreDelete = async () => { - const confirmPayload = { + const onLoaderDelete = (file, vectorStoreConfig, recordManagerConfig) => { + const props = { + title: `Delete`, + description: `Delete Loader ${file.loaderName} ? This will delete all the associated document chunks.`, + vectorStoreConfig, + recordManagerConfig, + type: 'LOADER' + } + + setDeleteDocStoreDialogProps(props) + setShowDeleteDocStoreDialog(true) + } + + const onStoreDelete = (vectorStoreConfig, recordManagerConfig) => { + const props = { title: `Delete`, description: `Delete Store ${getSpecificDocumentStore.data?.name} ? This will delete all the associated loaders and document chunks.`, - confirmButtonName: 'Delete', - cancelButtonName: 'Cancel' + vectorStoreConfig, + recordManagerConfig, + type: 'STORE' } - const isConfirmed = await confirm(confirmPayload) - if (isConfirmed) { - try { - const deleteResp = await documentsApi.deleteDocumentStore(storeId) - if (deleteResp.data) { - enqueueSnackbar({ - message: 'Store, Loader and associated document chunks deleted', - options: { - key: new Date().getTime() + Math.random(), - variant: 'success', - action: (key) => ( - - ) - } - }) - navigate('/document-stores/') - } - } catch (error) { - setError(error) - const errorData = error.response.data || `${error.response.status}: ${error.response.statusText}` - enqueueSnackbar({ - message: `Failed to delete loader: ${errorData}`, - options: { - key: new Date().getTime() + Math.random(), - variant: 'error', - persist: true, - action: (key) => ( - - ) - } - }) - } - } + setDeleteDocStoreDialogProps(props) + setShowDeleteDocStoreDialog(true) } const onEditClicked = () => { @@ -315,24 +356,15 @@ const DocumentStoreDetails = () => { onBack={() => navigate('/document-stores')} onEdit={() => onEditClicked()} > - + onStoreDelete(documentStore.vectorStoreConfig, documentStore.recordManagerConfig)} + size='small' + color='error' + title='Delete Document Store' + sx={{ mr: 2 }} + > - {documentStore?.status === 'STALE' && ( - - )} - {documentStore?.totalChunks > 0 && ( - - )} { > Add Document Loader + {(documentStore?.status === 'STALE' || documentStore?.status === 'UPSERTING') && ( + + )} + {documentStore?.status === 'UPSERTING' && ( + + )} + {documentStore?.totalChunks > 0 && documentStore?.status !== 'UPSERTING' && ( + <> + + + + )} + {documentStore?.totalChunks > 0 && documentStore?.status === 'UPSERTED' && ( + + )} {getSpecificDocumentStore.data?.whereUsed?.length > 0 && ( @@ -482,7 +575,13 @@ const DocumentStoreDetails = () => { theme={theme} onEditClick={() => openPreviewSettings(loader.id)} onViewChunksClick={() => showStoredChunks(loader.id)} - onDeleteClick={() => onLoaderDelete(loader)} + onDeleteClick={() => + onLoaderDelete( + loader, + documentStore?.vectorStoreConfig, + documentStore?.recordManagerConfig + ) + } /> ))} @@ -520,7 +619,15 @@ const DocumentStoreDetails = () => { onDocLoaderSelected={onDocLoaderSelected} /> )} - + {showDeleteDocStoreDialog && ( + setShowDeleteDocStoreDialog(false)} + onDelete={onDocStoreDelete} + /> + )} + {isBackdropLoading && } ) } diff --git a/packages/ui/src/views/docstore/DocumentStoreStatus.jsx b/packages/ui/src/views/docstore/DocumentStoreStatus.jsx index 1a4afbe9..e295fb76 100644 --- a/packages/ui/src/views/docstore/DocumentStoreStatus.jsx +++ b/packages/ui/src/views/docstore/DocumentStoreStatus.jsx @@ -15,8 +15,10 @@ const DocumentStoreStatus = ({ status, isTableView }) => { case 'EMPTY': return ['#673ab7', '#673ab7', '#673ab7'] case 'SYNCING': + case 'UPSERTING': return ['#fff8e1', '#ffe57f', '#ffc107'] case 'SYNC': + case 'UPSERTED': return ['#cdf5d8', '#00e676', '#00c853'] case 'NEW': return ['#e3f2fd', '#2196f3', '#1e88e5'] diff --git a/packages/ui/src/views/docstore/LoaderConfigPreviewChunks.jsx b/packages/ui/src/views/docstore/LoaderConfigPreviewChunks.jsx index 91d7d736..967f7e95 100644 --- a/packages/ui/src/views/docstore/LoaderConfigPreviewChunks.jsx +++ b/packages/ui/src/views/docstore/LoaderConfigPreviewChunks.jsx @@ -279,7 +279,6 @@ const LoaderConfigPreviewChunks = () => { useEffect(() => { if (getNodeDetailsApi.data) { const nodeData = cloneDeep(initNode(getNodeDetailsApi.data, uuidv4())) - // If this is a document store edit config, set the existing input values if (existingLoaderFromDocStoreTable && existingLoaderFromDocStoreTable.loaderConfig) { nodeData.inputs = existingLoaderFromDocStoreTable.loaderConfig diff --git a/packages/ui/src/views/docstore/UpsertHistoryDetailsDialog.jsx b/packages/ui/src/views/docstore/UpsertHistoryDetailsDialog.jsx new file mode 100644 index 00000000..8059f8b4 --- /dev/null +++ b/packages/ui/src/views/docstore/UpsertHistoryDetailsDialog.jsx @@ -0,0 +1,156 @@ +import { useState } from 'react' +import { createPortal } from 'react-dom' +import PropTypes from 'prop-types' +import { + Box, + Paper, + Accordion, + AccordionSummary, + AccordionDetails, + Dialog, + DialogContent, + DialogTitle, + Typography, + Table, + TableBody, + TableContainer, + TableRow, + TableCell +} from '@mui/material' +import ExpandMoreIcon from '@mui/icons-material/ExpandMore' +import { TableViewOnly } from '@/ui-component/table/Table' +import StatsCard from '@/ui-component/cards/StatsCard' + +// const +import { baseURL } from '@/store/constant' + +const UpsertHistoryDetailsDialog = ({ show, dialogProps, onCancel }) => { + const portalElement = document.getElementById('portal') + const [nodeConfigExpanded, setNodeConfigExpanded] = useState({}) + + const handleAccordionChange = (nodeLabel) => (event, isExpanded) => { + const accordianNodes = { ...nodeConfigExpanded } + accordianNodes[nodeLabel] = isExpanded + setNodeConfigExpanded(accordianNodes) + } + const component = show ? ( + + + {dialogProps.title} + + +
+ + + + +
+
+ + + + + + + {(dialogProps.flowData ?? []).map((node, index) => { + return ( + + } + aria-controls={`nodes-accordian-${node.name}`} + id={`nodes-accordian-header-${node.name}`} + > +
+
+ {node.name} +
+ {node.label} +
+ + {node.id} + +
+
+ + + {node.paramValues[0] && ( + + )} + + + ) + })} + + + + +
+ +
+ + + ) : null + + return createPortal(component, portalElement) +} + +UpsertHistoryDetailsDialog.propTypes = { + show: PropTypes.bool, + dialogProps: PropTypes.object, + onCancel: PropTypes.func +} + +export default UpsertHistoryDetailsDialog diff --git a/packages/ui/src/views/docstore/UpsertHistorySideDrawer.jsx b/packages/ui/src/views/docstore/UpsertHistorySideDrawer.jsx new file mode 100644 index 00000000..cc834648 --- /dev/null +++ b/packages/ui/src/views/docstore/UpsertHistorySideDrawer.jsx @@ -0,0 +1,113 @@ +import { useEffect, useState } from 'react' +import PropTypes from 'prop-types' +import moment from 'moment/moment' + +import { Stack, Button, Box, SwipeableDrawer } from '@mui/material' +import { IconSquareRoundedChevronsRight } from '@tabler/icons-react' +import { + Timeline, + TimelineConnector, + TimelineContent, + TimelineDot, + TimelineItem, + TimelineOppositeContent, + timelineOppositeContentClasses, + TimelineSeparator +} from '@mui/lab' +import HistoryEmptySVG from '@/assets/images/upsert_history_empty.svg' +import vectorstoreApi from '@/api/vectorstore' +import useApi from '@/hooks/useApi' + +const UpsertHistorySideDrawer = ({ show, dialogProps, onClickFunction, onSelectHistoryDetails }) => { + const onOpen = () => {} + const [upsertHistory, setUpsertHistory] = useState([]) + + const getUpsertHistoryApi = useApi(vectorstoreApi.getUpsertHistory) + + useEffect(() => { + getUpsertHistoryApi.request(dialogProps.id) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [dialogProps]) + + useEffect(() => { + if (getUpsertHistoryApi.data) { + setUpsertHistory(getUpsertHistoryApi.data) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [getUpsertHistoryApi.data]) + + return ( + <> + onClickFunction()} onOpen={onOpen}> + + + + {upsertHistory && + upsertHistory.map((history, index) => ( + + + {moment(history.date).format('DD-MMM-YYYY, hh:mm:ss A')} + + + + {index !== upsertHistory.length - 1 && } + + + {history.result.numAdded !== undefined && history.result.numAdded > 0 && ( + Added: {history.result.numAdded} + )} + {history.result.numUpdated !== undefined && history.result.numUpdated > 0 && ( + Updated: {history.result.numUpdated} + )} + {history.result.numSkipped !== undefined && history.result.numSkipped > 0 && ( + Skipped: {history.result.numSkipped} + )} + {history.result.numDeleted !== undefined && history.result.numDeleted > 0 && ( + Deleted: {history.result.numDeleted} + )} + + + + ))} + {upsertHistory.length === 0 && ( + + + HistoryEmptySVG + +
No Upsert History Yet
+ + )} + + + + + ) +} + +UpsertHistorySideDrawer.propTypes = { + show: PropTypes.bool, + dialogProps: PropTypes.object, + onClickFunction: PropTypes.func, + onSelectHistoryDetails: PropTypes.func +} + +export default UpsertHistorySideDrawer diff --git a/packages/ui/src/views/docstore/VectorStoreConfigure.jsx b/packages/ui/src/views/docstore/VectorStoreConfigure.jsx new file mode 100644 index 00000000..edecb844 --- /dev/null +++ b/packages/ui/src/views/docstore/VectorStoreConfigure.jsx @@ -0,0 +1,910 @@ +import { useEffect, useState } from 'react' +import { useDispatch, useSelector } from 'react-redux' +import { useNavigate } from 'react-router-dom' +import { cloneDeep } from 'lodash' +import { v4 as uuidv4 } from 'uuid' +import moment from 'moment/moment' + +// material-ui +import { Button, Stack, Grid, Box, Typography, IconButton, Stepper, Step, StepLabel } from '@mui/material' + +// project imports +import MainCard from '@/ui-component/cards/MainCard' +import ConfirmDialog from '@/ui-component/dialog/ConfirmDialog' +import ComponentsListDialog from '@/views/docstore/ComponentsListDialog' +import DocStoreInputHandler from '@/views/docstore/DocStoreInputHandler' +import ViewHeader from '@/layout/MainLayout/ViewHeader' +import { BackdropLoader } from '@/ui-component/loading/BackdropLoader' +import ErrorBoundary from '@/ErrorBoundary' +import UpsertResultDialog from '@/views/vectorstore/UpsertResultDialog' +import UpsertHistorySideDrawer from './UpsertHistorySideDrawer' +import UpsertHistoryDetailsDialog from './UpsertHistoryDetailsDialog' + +// API +import documentsApi from '@/api/documentstore' +import nodesApi from '@/api/nodes' + +// Hooks +import useApi from '@/hooks/useApi' + +// Store +import { closeSnackbar as closeSnackbarAction, enqueueSnackbar as enqueueSnackbarAction } from '@/store/actions' +import { baseURL } from '@/store/constant' + +// icons +import { IconX, IconEditCircle, IconRowInsertTop, IconDeviceFloppy, IconRefresh, IconClock } from '@tabler/icons-react' +import Embeddings from '@mui/icons-material/DynamicFeed' +import Storage from '@mui/icons-material/Storage' +import DynamicFeed from '@mui/icons-material/Filter1' + +// utils +import { initNode } from '@/utils/genericHelper' +import useNotifier from '@/utils/useNotifier' + +// const +const steps = ['Embeddings', 'Vector Store', 'Record Manager'] + +const VectorStoreConfigure = () => { + const navigate = useNavigate() + const dispatch = useDispatch() + useNotifier() + const customization = useSelector((state) => state.customization) + + const enqueueSnackbar = (...args) => dispatch(enqueueSnackbarAction(...args)) + const closeSnackbar = (...args) => dispatch(closeSnackbarAction(...args)) + + const getSpecificDocumentStoreApi = useApi(documentsApi.getSpecificDocumentStore) + const insertIntoVectorStoreApi = useApi(documentsApi.insertIntoVectorStore) + const saveVectorStoreConfigApi = useApi(documentsApi.saveVectorStoreConfig) + const getEmbeddingNodeDetailsApi = useApi(nodesApi.getSpecificNode) + const getVectorStoreNodeDetailsApi = useApi(nodesApi.getSpecificNode) + const getRecordManagerNodeDetailsApi = useApi(nodesApi.getSpecificNode) + + const [error, setError] = useState(null) + const [loading, setLoading] = useState(true) + + const [documentStore, setDocumentStore] = useState({}) + const [dialogProps, setDialogProps] = useState({}) + + const [showEmbeddingsListDialog, setShowEmbeddingsListDialog] = useState(false) + const [selectedEmbeddingsProvider, setSelectedEmbeddingsProvider] = useState({}) + + const [showVectorStoreListDialog, setShowVectorStoreListDialog] = useState(false) + const [selectedVectorStoreProvider, setSelectedVectorStoreProvider] = useState({}) + + const [showRecordManagerListDialog, setShowRecordManagerListDialog] = useState(false) + const [selectedRecordManagerProvider, setSelectedRecordManagerProvider] = useState({}) + const [isRecordManagerUnavailable, setRecordManagerUnavailable] = useState(false) + + const [showUpsertHistoryDialog, setShowUpsertHistoryDialog] = useState(false) + const [upsertResultDialogProps, setUpsertResultDialogProps] = useState({}) + + const [showUpsertHistorySideDrawer, setShowUpsertHistorySideDrawer] = useState(false) + const [upsertHistoryDrawerDialogProps, setUpsertHistoryDrawerDialogProps] = useState({}) + + const [showUpsertHistoryDetailsDialog, setShowUpsertHistoryDetailsDialog] = useState(false) + const [upsertDetailsDialogProps, setUpsertDetailsDialogProps] = useState({}) + + const onEmbeddingsSelected = (component) => { + const nodeData = cloneDeep(initNode(component, uuidv4())) + if (!showEmbeddingsListDialog && documentStore.embeddingConfig) { + nodeData.inputs = documentStore.embeddingConfig.config + nodeData.credential = documentStore.embeddingConfig.config.credential + } + setSelectedEmbeddingsProvider(nodeData) + setShowEmbeddingsListDialog(false) + } + + const showEmbeddingsList = () => { + const dialogProp = { + title: 'Select Embeddings Provider' + } + setDialogProps(dialogProp) + setShowEmbeddingsListDialog(true) + } + + const onVectorStoreSelected = (component) => { + const nodeData = cloneDeep(initNode(component, uuidv4())) + if (!nodeData.inputAnchors.find((anchor) => anchor.name === 'recordManager')) { + setRecordManagerUnavailable(true) + setSelectedRecordManagerProvider({}) + } else { + setRecordManagerUnavailable(false) + } + if (!showVectorStoreListDialog && documentStore.vectorStoreConfig) { + nodeData.inputs = documentStore.vectorStoreConfig.config + nodeData.credential = documentStore.vectorStoreConfig.config.credential + } + setSelectedVectorStoreProvider(nodeData) + setShowVectorStoreListDialog(false) + } + + const showVectorStoreList = () => { + const dialogProp = { + title: 'Select a Vector Store Provider' + } + setDialogProps(dialogProp) + setShowVectorStoreListDialog(true) + } + + const onRecordManagerSelected = (component) => { + const nodeData = cloneDeep(initNode(component, uuidv4())) + if (!showRecordManagerListDialog && documentStore.recordManagerConfig) { + nodeData.inputs = documentStore.recordManagerConfig.config + nodeData.credential = documentStore.recordManagerConfig.config.credential + } + setSelectedRecordManagerProvider(nodeData) + setShowRecordManagerListDialog(false) + } + + const showRecordManagerList = () => { + const dialogProp = { + title: 'Select a Record Manager' + } + setDialogProps(dialogProp) + setShowRecordManagerListDialog(true) + } + + const showUpsertHistoryDrawer = () => { + const dialogProp = { + id: storeId + } + setUpsertHistoryDrawerDialogProps(dialogProp) + setShowUpsertHistorySideDrawer(true) + } + + const onSelectHistoryDetails = (history) => { + const props = { + title: moment(history.date).format('DD-MMM-YYYY, hh:mm:ss A'), + numAdded: history.result.numAdded, + numUpdated: history.result.numUpdated, + numSkipped: history.result.numSkipped, + numDeleted: history.result.numDeleted, + flowData: history.flowData + } + setUpsertDetailsDialogProps(props) + setShowUpsertHistoryDetailsDialog(true) + } + + const checkMandatoryFields = () => { + let canSubmit = true + const inputParams = (selectedVectorStoreProvider?.inputParams ?? []).filter((inputParam) => !inputParam.hidden) + for (const inputParam of inputParams) { + if (!inputParam.optional && (!selectedVectorStoreProvider.inputs[inputParam.name] || !selectedVectorStoreProvider.credential)) { + if (inputParam.type === 'credential' && !selectedVectorStoreProvider.credential) { + canSubmit = false + break + } else if (inputParam.type !== 'credential' && !selectedVectorStoreProvider.inputs[inputParam.name]) { + canSubmit = false + break + } + } + } + + const inputParams2 = (selectedEmbeddingsProvider?.inputParams ?? []).filter((inputParam) => !inputParam.hidden) + for (const inputParam of inputParams2) { + if (!inputParam.optional && (!selectedEmbeddingsProvider.inputs[inputParam.name] || !selectedEmbeddingsProvider.credential)) { + if (inputParam.type === 'credential' && !selectedEmbeddingsProvider.credential) { + canSubmit = false + break + } else if (inputParam.type !== 'credential' && !selectedEmbeddingsProvider.inputs[inputParam.name]) { + canSubmit = false + break + } + } + } + + if (!canSubmit) { + enqueueSnackbar({ + message: 'Please fill in all mandatory fields.', + options: { + key: new Date().getTime() + Math.random(), + variant: 'warning', + action: (key) => ( + + ) + } + }) + } + return canSubmit + } + + const prepareConfigData = () => { + const data = { + storeId: storeId + } + // Set embedding config + if (selectedEmbeddingsProvider.inputs) { + data.embeddingConfig = {} + data.embeddingName = selectedEmbeddingsProvider.name + Object.keys(selectedEmbeddingsProvider.inputs).map((key) => { + if (key === 'FLOWISE_CREDENTIAL_ID') { + data.embeddingConfig['credential'] = selectedEmbeddingsProvider.inputs[key] + } else { + data.embeddingConfig[key] = selectedEmbeddingsProvider.inputs[key] + } + }) + } else { + data.embeddingConfig = null + data.embeddingName = '' + } + + // Set vector store config + if (selectedVectorStoreProvider.inputs) { + data.vectorStoreConfig = {} + data.vectorStoreName = selectedVectorStoreProvider.name + Object.keys(selectedVectorStoreProvider.inputs).map((key) => { + if (key === 'FLOWISE_CREDENTIAL_ID') { + data.vectorStoreConfig['credential'] = selectedVectorStoreProvider.inputs[key] + } else { + data.vectorStoreConfig[key] = selectedVectorStoreProvider.inputs[key] + } + }) + } else { + data.vectorStoreConfig = null + data.vectorStoreName = '' + } + + // Set record manager config + if (selectedRecordManagerProvider.inputs) { + data.recordManagerConfig = {} + data.recordManagerName = selectedRecordManagerProvider.name + Object.keys(selectedRecordManagerProvider.inputs).map((key) => { + if (key === 'FLOWISE_CREDENTIAL_ID') { + data.recordManagerConfig['credential'] = selectedRecordManagerProvider.inputs[key] + } else { + data.recordManagerConfig[key] = selectedRecordManagerProvider.inputs[key] + } + }) + } else { + data.recordManagerConfig = null + data.recordManagerName = '' + } + + return data + } + + const tryAndInsertIntoStore = () => { + if (checkMandatoryFields()) { + setLoading(true) + const data = prepareConfigData() + insertIntoVectorStoreApi.request(data) + } + } + + const saveVectorStoreConfig = () => { + setLoading(true) + const data = prepareConfigData() + saveVectorStoreConfigApi.request(data) + } + + const resetVectorStoreConfig = () => { + setSelectedEmbeddingsProvider({}) + setSelectedVectorStoreProvider({}) + setSelectedRecordManagerProvider({}) + } + + const getActiveStep = () => { + if (selectedRecordManagerProvider && Object.keys(selectedRecordManagerProvider).length > 0) { + return 3 + } + if (selectedVectorStoreProvider && Object.keys(selectedVectorStoreProvider).length > 0) { + return 2 + } + if (selectedEmbeddingsProvider && Object.keys(selectedEmbeddingsProvider).length > 0) { + return 1 + } + return 0 + } + + const Steps = () => { + return ( + + + {steps.map((label) => ( + + {label} + + ))} + + + ) + } + + const isRecordManagerDisabled = () => { + return Object.keys(selectedVectorStoreProvider).length === 0 || isRecordManagerUnavailable + } + + const isVectorStoreDisabled = () => { + return Object.keys(selectedEmbeddingsProvider).length === 0 + } + + useEffect(() => { + if (saveVectorStoreConfigApi.data) { + setLoading(false) + enqueueSnackbar({ + message: 'Configuration saved successfully', + options: { + key: new Date().getTime() + Math.random(), + variant: 'success', + action: (key) => ( + + ) + } + }) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [saveVectorStoreConfigApi.data]) + + useEffect(() => { + if (insertIntoVectorStoreApi.data) { + setLoading(false) + setShowUpsertHistoryDialog(true) + setUpsertResultDialogProps({ ...insertIntoVectorStoreApi.data, goToRetrievalQuery: true }) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [insertIntoVectorStoreApi.data]) + + useEffect(() => { + if (insertIntoVectorStoreApi.error) { + setLoading(false) + setError(insertIntoVectorStoreApi.error) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [insertIntoVectorStoreApi.error]) + + useEffect(() => { + if (saveVectorStoreConfigApi.error) { + setLoading(false) + setError(saveVectorStoreConfigApi.error) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [saveVectorStoreConfigApi.error]) + + const URLpath = document.location.pathname.toString().split('/') + const storeId = URLpath[URLpath.length - 1] === 'document-stores' ? '' : URLpath[URLpath.length - 1] + useEffect(() => { + getSpecificDocumentStoreApi.request(storeId) + + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []) + + useEffect(() => { + if (getSpecificDocumentStoreApi.data) { + const docStore = getSpecificDocumentStoreApi.data + setDocumentStore(docStore) + if (docStore.embeddingConfig) { + getEmbeddingNodeDetailsApi.request(docStore.embeddingConfig.name) + } + if (docStore.vectorStoreConfig) { + getVectorStoreNodeDetailsApi.request(docStore.vectorStoreConfig.name) + } + if (docStore.recordManagerConfig) { + getRecordManagerNodeDetailsApi.request(docStore.recordManagerConfig.name) + } + setLoading(false) + } + + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [getSpecificDocumentStoreApi.data]) + + useEffect(() => { + if (getEmbeddingNodeDetailsApi.data) { + const node = getEmbeddingNodeDetailsApi.data + onEmbeddingsSelected(node) + } + + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [getEmbeddingNodeDetailsApi.data]) + + useEffect(() => { + if (getVectorStoreNodeDetailsApi.data) { + const node = getVectorStoreNodeDetailsApi.data + onVectorStoreSelected(node) + } + + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [getVectorStoreNodeDetailsApi.data]) + + useEffect(() => { + if (getRecordManagerNodeDetailsApi.data) { + const node = getRecordManagerNodeDetailsApi.data + onRecordManagerSelected(node) + } + + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [getRecordManagerNodeDetailsApi.data]) + + useEffect(() => { + if (getSpecificDocumentStoreApi.error) { + setError(getSpecificDocumentStoreApi.error) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [getSpecificDocumentStoreApi.error]) + + return ( + <> + + {error ? ( + + ) : ( + + navigate(-1)} + > + {(Object.keys(selectedEmbeddingsProvider).length > 0 || + Object.keys(selectedVectorStoreProvider).length > 0) && ( + + )} + {(Object.keys(selectedEmbeddingsProvider).length > 0 || + Object.keys(selectedVectorStoreProvider).length > 0) && ( + + )} + {Object.keys(selectedEmbeddingsProvider).length > 0 && Object.keys(selectedVectorStoreProvider).length > 0 && ( + + )} + + + + + + + + {Object.keys(selectedEmbeddingsProvider).length === 0 ? ( + + ) : ( + + + +
+ +
+ {selectedEmbeddingsProvider.label ? ( + {selectedEmbeddingsProvider.label + ) : ( + + )} +
+ + {selectedEmbeddingsProvider.label} + +
+
+ {Object.keys(selectedEmbeddingsProvider).length > 0 && ( + <> + + + + + )} +
+ + {selectedEmbeddingsProvider && + Object.keys(selectedEmbeddingsProvider).length > 0 && + (selectedEmbeddingsProvider.inputParams ?? []) + .filter((inputParam) => !inputParam.hidden) + .map((inputParam, index) => ( + + ))} +
+ + + + )} + + + {Object.keys(selectedVectorStoreProvider).length === 0 ? ( + + ) : ( + + + +
+ +
+ {selectedVectorStoreProvider.label ? ( + {selectedVectorStoreProvider.label + ) : ( + + )} +
+ + {selectedVectorStoreProvider.label} + +
+
+ {Object.keys(selectedVectorStoreProvider).length > 0 && ( + <> + + + + + )} +
+ + {selectedVectorStoreProvider && + Object.keys(selectedVectorStoreProvider).length > 0 && + (selectedVectorStoreProvider.inputParams ?? []) + .filter((inputParam) => !inputParam.hidden) + .map((inputParam, index) => ( + + ))} +
+ + + + )} + + + {Object.keys(selectedRecordManagerProvider).length === 0 ? ( + + ) : ( + + + +
+ +
+ {selectedRecordManagerProvider.label ? ( + {selectedRecordManagerProvider.label + ) : ( + + )} +
+ + {selectedRecordManagerProvider.label} + +
+
+ {Object.keys(selectedRecordManagerProvider).length > 0 && ( + <> + + + + + )} +
+ + {selectedRecordManagerProvider && + Object.keys(selectedRecordManagerProvider).length > 0 && + (selectedRecordManagerProvider.inputParams ?? []) + .filter((inputParam) => !inputParam.hidden) + .map((inputParam, index) => ( + <> + + + ))} +
+ + + + )} + + + + )} + + + {showEmbeddingsListDialog && ( + setShowEmbeddingsListDialog(false)} + apiCall={documentsApi.getEmbeddingProviders} + onSelected={onEmbeddingsSelected} + /> + )} + {showVectorStoreListDialog && ( + setShowVectorStoreListDialog(false)} + apiCall={documentsApi.getVectorStoreProviders} + onSelected={onVectorStoreSelected} + /> + )} + {showRecordManagerListDialog && ( + setShowRecordManagerListDialog(false)} + apiCall={documentsApi.getRecordManagerProviders} + onSelected={onRecordManagerSelected} + /> + )} + {showUpsertHistoryDialog && ( + { + setShowUpsertHistoryDialog(false) + }} + onGoToRetrievalQuery={() => navigate('/document-stores/query/' + storeId)} + > + )} + {showUpsertHistorySideDrawer && ( + setShowUpsertHistorySideDrawer(false)} + onSelectHistoryDetails={onSelectHistoryDetails} + /> + )} + {showUpsertHistoryDetailsDialog && ( + setShowUpsertHistoryDetailsDialog(false)} + /> + )} + + {loading && } + + ) +} + +export default VectorStoreConfigure diff --git a/packages/ui/src/views/docstore/VectorStoreQuery.jsx b/packages/ui/src/views/docstore/VectorStoreQuery.jsx new file mode 100644 index 00000000..69234ed4 --- /dev/null +++ b/packages/ui/src/views/docstore/VectorStoreQuery.jsx @@ -0,0 +1,483 @@ +import { useEffect, useState, useRef } from 'react' +import { useDispatch, useSelector } from 'react-redux' +import { useNavigate } from 'react-router-dom' +import ReactJson from 'flowise-react-json-view' +import { cloneDeep } from 'lodash' +import { v4 as uuidv4 } from 'uuid' + +// material-ui +import { Box, Card, Grid, Stack, Typography, OutlinedInput, IconButton, Button } from '@mui/material' +import Embeddings from '@mui/icons-material/DynamicFeed' +import { useTheme, styled } from '@mui/material/styles' +import CardContent from '@mui/material/CardContent' +import chunks_emptySVG from '@/assets/images/chunks_empty.svg' +import { IconSearch, IconFileStack, IconDeviceFloppy, IconX } from '@tabler/icons-react' + +// project imports +import MainCard from '@/ui-component/cards/MainCard' +import { BackdropLoader } from '@/ui-component/loading/BackdropLoader' +import ConfirmDialog from '@/ui-component/dialog/ConfirmDialog' +import ExpandedChunkDialog from './ExpandedChunkDialog' +import ViewHeader from '@/layout/MainLayout/ViewHeader' +import DocStoreInputHandler from '@/views/docstore/DocStoreInputHandler' + +// API +import documentsApi from '@/api/documentstore' +import nodesApi from '@/api/nodes' + +// Hooks +import useApi from '@/hooks/useApi' +import useNotifier from '@/utils/useNotifier' +import { baseURL } from '@/store/constant' +import { initNode } from '@/utils/genericHelper' +import { closeSnackbar as closeSnackbarAction, enqueueSnackbar as enqueueSnackbarAction } from '@/store/actions' + +const CardWrapper = styled(MainCard)(({ theme }) => ({ + background: theme.palette.card.main, + color: theme.darkTextPrimary, + overflow: 'auto', + position: 'relative', + boxShadow: '0 2px 14px 0 rgb(32 40 45 / 8%)', + cursor: 'pointer', + '&:hover': { + background: theme.palette.card.hover, + boxShadow: '0 2px 14px 0 rgb(32 40 45 / 20%)' + }, + maxHeight: '250px', + minHeight: '250px', + maxWidth: '100%', + overflowWrap: 'break-word', + whiteSpace: 'pre-line', + padding: 1 +})) + +const VectorStoreQuery = () => { + const customization = useSelector((state) => state.customization) + const navigate = useNavigate() + const theme = useTheme() + const dispatch = useDispatch() + const inputRef = useRef(null) + + useNotifier() + + const enqueueSnackbar = (...args) => dispatch(enqueueSnackbarAction(...args)) + const closeSnackbar = (...args) => dispatch(closeSnackbarAction(...args)) + + const URLpath = document.location.pathname.toString().split('/') + const storeId = URLpath[URLpath.length - 1] === 'document-stores' ? '' : URLpath[URLpath.length - 1] + + const [documentChunks, setDocumentChunks] = useState([]) + const [loading, setLoading] = useState(false) + const [showExpandedChunkDialog, setShowExpandedChunkDialog] = useState(false) + const [expandedChunkDialogProps, setExpandedChunkDialogProps] = useState({}) + const [documentStore, setDocumentStore] = useState({}) + const [query, setQuery] = useState('') + + const [timeTaken, setTimeTaken] = useState(-1) + const [retrievalError, setRetrievalError] = useState(undefined) + + const getSpecificDocumentStoreApi = useApi(documentsApi.getSpecificDocumentStore) + const queryVectorStoreApi = useApi(documentsApi.queryVectorStore) + + const getVectorStoreNodeDetailsApi = useApi(nodesApi.getSpecificNode) + const [selectedVectorStoreProvider, setSelectedVectorStoreProvider] = useState({}) + + const chunkSelected = (chunkId, selectedChunkNumber) => { + const selectedChunk = documentChunks.find((chunk) => chunk.id === chunkId) + const dialogProps = { + data: { + selectedChunk, + selectedChunkNumber + } + } + setExpandedChunkDialogProps(dialogProps) + setShowExpandedChunkDialog(true) + } + + const handleEnter = (e) => { + // Check if IME composition is in progress + const isIMEComposition = e.isComposing || e.keyCode === 229 + if (e.key === 'Enter' && query && !isIMEComposition) { + if (!e.shiftKey && query) { + if (inputRef.current) { + inputRef.current.blur() + } + doQuery() + } + } else if (e.key === 'Enter') { + e.preventDefault() + } + } + + const doQuery = () => { + setLoading(true) + const data = { + query: query, + storeId: storeId, + inputs: selectedVectorStoreProvider.inputs + } + queryVectorStoreApi.request(data) + } + + const saveConfig = async () => { + setLoading(true) + const data = { + storeId: storeId + } + + if (selectedVectorStoreProvider.inputs) { + data.vectorStoreConfig = {} + data.vectorStoreName = selectedVectorStoreProvider.name + Object.keys(selectedVectorStoreProvider.inputs).map((key) => { + if (key === 'FLOWISE_CREDENTIAL_ID') { + data.vectorStoreConfig['credential'] = selectedVectorStoreProvider.inputs[key] + } else { + data.vectorStoreConfig[key] = selectedVectorStoreProvider.inputs[key] + } + }) + } + + try { + const updateResp = await documentsApi.updateVectorStoreConfig(data) + setLoading(false) + if (updateResp.data) { + enqueueSnackbar({ + message: 'Vector Store Config Successfully Updated', + options: { + key: new Date().getTime() + Math.random(), + variant: 'success', + action: (key) => ( + + ) + } + }) + } + } catch (error) { + setLoading(false) + const errorData = error.response?.data || `${error.response?.status}: ${error.response?.statusText}` + enqueueSnackbar({ + message: `Failed to update vector store config: ${errorData}`, + options: { + key: new Date().getTime() + Math.random(), + variant: 'error', + persist: true, + action: (key) => ( + + ) + } + }) + } + } + + useEffect(() => { + if (queryVectorStoreApi.data) { + setDocumentChunks(queryVectorStoreApi.data.docs) + setTimeTaken(queryVectorStoreApi.data.timeTaken) + setRetrievalError(undefined) + setLoading(false) + if (inputRef.current) { + inputRef.current.focus() + } + } + + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [queryVectorStoreApi.data]) + + useEffect(() => { + if (queryVectorStoreApi.error) { + if (queryVectorStoreApi.error.response?.data?.message) { + const message = queryVectorStoreApi.error.response.data.message + // remove the text 'documentStoreServices.queryVectorStore - ' from the error message to make it readable + setRetrievalError(message.replace('documentStoreServices.queryVectorStore - ', '')) + setDocumentChunks([]) + setTimeTaken(-1) + } + setLoading(false) + } + + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [queryVectorStoreApi.error]) + + useEffect(() => { + if (getVectorStoreNodeDetailsApi.data) { + const node = getVectorStoreNodeDetailsApi.data + fetchVectorStoreDetails(node) + } + + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [getVectorStoreNodeDetailsApi.data]) + + const fetchVectorStoreDetails = (component) => { + const nodeData = cloneDeep(initNode(component, uuidv4())) + if (documentStore.vectorStoreConfig) { + nodeData.inputs = documentStore.vectorStoreConfig.config + nodeData.credential = documentStore.vectorStoreConfig.config.credential + } + setSelectedVectorStoreProvider(nodeData) + } + + useEffect(() => { + getSpecificDocumentStoreApi.request(storeId) + + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []) + + useEffect(() => { + if (getSpecificDocumentStoreApi.data) { + setDocumentStore(getSpecificDocumentStoreApi.data) + const vectorStoreConfig = getSpecificDocumentStoreApi.data.vectorStoreConfig + if (vectorStoreConfig) { + getVectorStoreNodeDetailsApi.request(vectorStoreConfig.name) + } + setLoading(false) + } + + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [getSpecificDocumentStoreApi.data]) + + return ( + <> + + + navigate(-1)} + > + + +
+
+ + + +
+ + Enter your Query * + + +
+
+ setQuery(e.target.value)} + onKeyDown={handleEnter} + value={query ?? ''} + endAdornment={ + + + + } + /> + + + + + + + +
+ +
+ {selectedVectorStoreProvider.label ? ( + {selectedVectorStoreProvider.label + ) : ( + + )} +
+ + {selectedVectorStoreProvider.label} + +
+ + {selectedVectorStoreProvider && + Object.keys(selectedVectorStoreProvider).length > 0 && + (selectedVectorStoreProvider.inputParams ?? []) + .filter((inputParam) => !inputParam.hidden) + .map((inputParam, index) => ( + + ))} +
+ + + + + + +
+ +
+ + Retrieved Documents + {timeTaken > -1 && ( + + Count: {documentChunks.length}. Time taken: {timeTaken} millis. + + )} + {retrievalError && ( + + {retrievalError} + + )} + +
+ + {!documentChunks.length && ( +
+ + chunks_emptySVG + +
No Documents Retrieved
+
+ )} + + {documentChunks?.length > 0 && + documentChunks.map((row, index) => ( + + chunkSelected(row.id, row.chunkNo)} + sx={{ border: 1, borderColor: theme.palette.grey[900] + 25, borderRadius: 2 }} + > + + + + {`#${row.chunkNo}. Characters: ${row.pageContent.length}`} + + + {row.pageContent} + + + + + + + ))} + + + + +
+ + + + setShowExpandedChunkDialog(false)} + isReadOnly={true} + > + {loading && } + + ) +} + +export default VectorStoreQuery diff --git a/packages/ui/src/views/vectorstore/UpsertResultDialog.jsx b/packages/ui/src/views/vectorstore/UpsertResultDialog.jsx index 4276bd8a..e2b4a37e 100644 --- a/packages/ui/src/views/vectorstore/UpsertResultDialog.jsx +++ b/packages/ui/src/views/vectorstore/UpsertResultDialog.jsx @@ -6,8 +6,9 @@ import ReactJson from 'flowise-react-json-view' import { Typography, Card, CardContent, Button, Dialog, DialogActions, DialogContent, DialogTitle } from '@mui/material' import StatsCard from '@/ui-component/cards/StatsCard' import { HIDE_CANVAS_DIALOG, SHOW_CANVAS_DIALOG } from '@/store/actions' +import { IconZoomScan } from '@tabler/icons-react' -const UpsertResultDialog = ({ show, dialogProps, onCancel }) => { +const UpsertResultDialog = ({ show, dialogProps, onCancel, onGoToRetrievalQuery }) => { const portalElement = document.getElementById('portal') const dispatch = useDispatch() const customization = useSelector((state) => state.customization) @@ -76,7 +77,31 @@ const UpsertResultDialog = ({ show, dialogProps, onCancel }) => { - + {dialogProps.goToRetrievalQuery && ( +
+ + +
+ )} + {!dialogProps.goToRetrievalQuery && } ) : null @@ -88,7 +113,7 @@ UpsertResultDialog.propTypes = { show: PropTypes.bool, dialogProps: PropTypes.object, onCancel: PropTypes.func, - onSave: PropTypes.func + onGoToRetrievalQuery: PropTypes.func } export default UpsertResultDialog