mirror of
https://github.com/farcasclaudiu/Flowise.git
synced 2026-06-28 09:00:52 +03:00
Document Store - Phase 2 (#2912)
* Document Store - Phase 2 * Adding additional columns for vector store config, document store phase 2 * Adding additional columns for vector store config, document store phase 2 * Document Store - Phase 2 - Upsert and Query * ux cleanup * retrieval settings and more ux changes * adding MMR params to execution * Making the upsert process async. * add upsert history changes * making the searchParams dynamic * removing unnecessary params * add ability to delete data from vector store * update margin for vector store query * adding option to save config in the retrieval playground * adding chunk number for query return chunks * Adding a Document Store node in the VectorStore category * update doc store status, ui touchup --------- Co-authored-by: Henry <hzj94@hotmail.com>
This commit is contained in:
@@ -150,6 +150,42 @@ class Chroma_VectorStores implements INode {
|
||||
} catch (e) {
|
||||
throw new Error(e)
|
||||
}
|
||||
},
|
||||
async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise<void> {
|
||||
const collectionName = nodeData.inputs?.collectionName as string
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const chromaURL = nodeData.inputs?.chromaURL as string
|
||||
const recordManager = nodeData.inputs?.recordManager
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const chromaApiKey = getCredentialParam('chromaApiKey', credentialData, nodeData)
|
||||
|
||||
const obj: {
|
||||
collectionName: string
|
||||
url?: string
|
||||
chromaApiKey?: string
|
||||
} = { collectionName }
|
||||
if (chromaURL) obj.url = chromaURL
|
||||
if (chromaApiKey) obj.chromaApiKey = chromaApiKey
|
||||
|
||||
try {
|
||||
if (recordManager) {
|
||||
const vectorStoreName = collectionName
|
||||
await recordManager.createSchema()
|
||||
;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
|
||||
const keys: string[] = await recordManager.listKeys({})
|
||||
|
||||
const chromaStore = new ChromaExtended(embeddings, obj)
|
||||
|
||||
await chromaStore.delete({ ids: keys })
|
||||
await recordManager.deleteKeys(keys)
|
||||
} else {
|
||||
const chromaStore = new ChromaExtended(embeddings, obj)
|
||||
await chromaStore.delete({ ids })
|
||||
}
|
||||
} catch (e) {
|
||||
throw new Error(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
import { ICommonObject, IDatabaseEntity, INode, INodeData, INodeOptionsValue, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { DataSource } from 'typeorm'
|
||||
|
||||
class DocStore_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
outputs: INodeOutputsValue[]
|
||||
badge: string
|
||||
|
||||
constructor() {
|
||||
this.label = 'Document Store (Vector)'
|
||||
this.name = 'documentStoreVS'
|
||||
this.version = 1.0
|
||||
this.type = 'DocumentStoreVS'
|
||||
this.icon = 'dstore.svg'
|
||||
this.badge = 'New'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = `Search and retrieve documents from Document Store`
|
||||
this.baseClasses = [this.type]
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Select Store',
|
||||
name: 'selectedStore',
|
||||
type: 'asyncOptions',
|
||||
loadMethod: 'listStores'
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: ['BaseRetriever']
|
||||
},
|
||||
{
|
||||
label: 'Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: ['VectorStore']
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
//@ts-ignore
|
||||
loadMethods = {
|
||||
async listStores(_: INodeData, options: ICommonObject): Promise<INodeOptionsValue[]> {
|
||||
const returnData: INodeOptionsValue[] = []
|
||||
|
||||
const appDataSource = options.appDataSource as DataSource
|
||||
const databaseEntities = options.databaseEntities as IDatabaseEntity
|
||||
|
||||
if (appDataSource === undefined || !appDataSource) {
|
||||
return returnData
|
||||
}
|
||||
|
||||
const stores = await appDataSource.getRepository(databaseEntities['DocumentStore']).find()
|
||||
for (const store of stores) {
|
||||
if (store.status === 'UPSERTED') {
|
||||
const obj = {
|
||||
name: store.id,
|
||||
label: store.name,
|
||||
description: store.description
|
||||
}
|
||||
returnData.push(obj)
|
||||
}
|
||||
}
|
||||
return returnData
|
||||
}
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const selectedStore = nodeData.inputs?.selectedStore as string
|
||||
const appDataSource = options.appDataSource as DataSource
|
||||
const databaseEntities = options.databaseEntities as IDatabaseEntity
|
||||
const output = nodeData.outputs?.output as string
|
||||
|
||||
const entity = await appDataSource.getRepository(databaseEntities['DocumentStore']).findOneBy({ id: selectedStore })
|
||||
if (!entity) {
|
||||
return { error: 'Store not found' }
|
||||
}
|
||||
const data: ICommonObject = {}
|
||||
data.output = output
|
||||
|
||||
// Prepare Embeddings Instance
|
||||
const embeddingConfig = JSON.parse(entity.embeddingConfig)
|
||||
data.embeddingName = embeddingConfig.name
|
||||
data.embeddingConfig = embeddingConfig.config
|
||||
let embeddingObj = await _createEmbeddingsObject(options.componentNodes, data, options)
|
||||
if (!embeddingObj) {
|
||||
return { error: 'Failed to create EmbeddingObj' }
|
||||
}
|
||||
|
||||
// Prepare Vector Store Instance
|
||||
const vsConfig = JSON.parse(entity.vectorStoreConfig)
|
||||
data.vectorStoreName = vsConfig.name
|
||||
data.vectorStoreConfig = vsConfig.config
|
||||
if (data.inputs) {
|
||||
data.vectorStoreConfig = { ...vsConfig.config, ...data.inputs }
|
||||
}
|
||||
|
||||
// Prepare Vector Store Node Data
|
||||
const vStoreNodeData = _createVectorStoreNodeData(options.componentNodes, data, embeddingObj)
|
||||
|
||||
// Finally create the Vector Store or Retriever object (data.output)
|
||||
const vectorStoreObj = await _createVectorStoreObject(options.componentNodes, data)
|
||||
const retrieverOrVectorStore = await vectorStoreObj.init(vStoreNodeData, '', options)
|
||||
if (!retrieverOrVectorStore) {
|
||||
return { error: 'Failed to create vectorStore' }
|
||||
}
|
||||
return retrieverOrVectorStore
|
||||
}
|
||||
}
|
||||
|
||||
const _createEmbeddingsObject = async (componentNodes: ICommonObject, data: ICommonObject, options: ICommonObject): Promise<any> => {
|
||||
// prepare embedding node data
|
||||
const embeddingComponent = componentNodes[data.embeddingName]
|
||||
const embeddingNodeData: any = {
|
||||
inputs: { ...data.embeddingConfig },
|
||||
outputs: { output: 'document' },
|
||||
id: `${embeddingComponent.name}_0`,
|
||||
label: embeddingComponent.label,
|
||||
name: embeddingComponent.name,
|
||||
category: embeddingComponent.category,
|
||||
inputParams: embeddingComponent.inputs || []
|
||||
}
|
||||
if (data.embeddingConfig.credential) {
|
||||
embeddingNodeData.credential = data.embeddingConfig.credential
|
||||
}
|
||||
|
||||
// init embedding object
|
||||
const embeddingNodeInstanceFilePath = embeddingComponent.filePath as string
|
||||
const embeddingNodeModule = await import(embeddingNodeInstanceFilePath)
|
||||
const embeddingNodeInstance = new embeddingNodeModule.nodeClass()
|
||||
return await embeddingNodeInstance.init(embeddingNodeData, '', options)
|
||||
}
|
||||
|
||||
const _createVectorStoreNodeData = (componentNodes: ICommonObject, data: ICommonObject, embeddingObj: any) => {
|
||||
const vectorStoreComponent = componentNodes[data.vectorStoreName]
|
||||
const vStoreNodeData: any = {
|
||||
id: `${vectorStoreComponent.name}_0`,
|
||||
inputs: { ...data.vectorStoreConfig },
|
||||
outputs: { output: data.output },
|
||||
label: vectorStoreComponent.label,
|
||||
name: vectorStoreComponent.name,
|
||||
category: vectorStoreComponent.category
|
||||
}
|
||||
if (data.vectorStoreConfig.credential) {
|
||||
vStoreNodeData.credential = data.vectorStoreConfig.credential
|
||||
}
|
||||
|
||||
if (embeddingObj) {
|
||||
vStoreNodeData.inputs.embeddings = embeddingObj
|
||||
}
|
||||
|
||||
// Get all input params except the ones that are anchor points to avoid JSON stringify circular error
|
||||
const filterInputParams = ['document', 'embeddings', 'recordManager']
|
||||
const inputParams = vectorStoreComponent.inputs?.filter((input: any) => !filterInputParams.includes(input.name))
|
||||
vStoreNodeData.inputParams = inputParams
|
||||
return vStoreNodeData
|
||||
}
|
||||
|
||||
const _createVectorStoreObject = async (componentNodes: ICommonObject, data: ICommonObject) => {
|
||||
const vStoreNodeInstanceFilePath = componentNodes[data.vectorStoreName].filePath as string
|
||||
const vStoreNodeModule = await import(vStoreNodeInstanceFilePath)
|
||||
const vStoreNodeInstance = new vStoreNodeModule.nodeClass()
|
||||
return vStoreNodeInstance
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: DocStore_VectorStores }
|
||||
@@ -0,0 +1,15 @@
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
width="24"
|
||||
height="24"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
>
|
||||
<path d="M12 4l-8 4l8 4l8 -4l-8 -4" />
|
||||
<path d="M4 12l8 4l8 -4" />
|
||||
<path d="M4 16l8 4l8 -4" />
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 305 B |
@@ -163,6 +163,35 @@ class Elasticsearch_VectorStores implements INode {
|
||||
} catch (e) {
|
||||
throw new Error(e)
|
||||
}
|
||||
},
|
||||
async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise<void> {
|
||||
const indexName = nodeData.inputs?.indexName as string
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const similarityMeasure = nodeData.inputs?.similarityMeasure as string
|
||||
const recordManager = nodeData.inputs?.recordManager
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const endPoint = getCredentialParam('endpoint', credentialData, nodeData)
|
||||
const cloudId = getCredentialParam('cloudId', credentialData, nodeData)
|
||||
|
||||
const elasticSearchClientArgs = prepareClientArgs(endPoint, cloudId, credentialData, nodeData, similarityMeasure, indexName)
|
||||
const vectorStore = new ElasticVectorSearch(embeddings, elasticSearchClientArgs)
|
||||
|
||||
try {
|
||||
if (recordManager) {
|
||||
const vectorStoreName = indexName
|
||||
await recordManager.createSchema()
|
||||
;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
|
||||
const keys: string[] = await recordManager.listKeys({})
|
||||
|
||||
await vectorStore.delete({ ids: keys })
|
||||
await recordManager.deleteKeys(keys)
|
||||
} else {
|
||||
await vectorStore.delete({ ids })
|
||||
}
|
||||
} catch (e) {
|
||||
throw new Error(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -184,6 +184,45 @@ class Pinecone_VectorStores implements INode {
|
||||
} catch (e) {
|
||||
throw new Error(e)
|
||||
}
|
||||
},
|
||||
async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise<void> {
|
||||
const _index = nodeData.inputs?.pineconeIndex as string
|
||||
const pineconeNamespace = nodeData.inputs?.pineconeNamespace as string
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const pineconeTextKey = nodeData.inputs?.pineconeTextKey as string
|
||||
const recordManager = nodeData.inputs?.recordManager
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const pineconeApiKey = getCredentialParam('pineconeApiKey', credentialData, nodeData)
|
||||
|
||||
const client = getPineconeClient({ apiKey: pineconeApiKey })
|
||||
|
||||
const pineconeIndex = client.Index(_index)
|
||||
|
||||
const obj: PineconeStoreParams = {
|
||||
pineconeIndex,
|
||||
textKey: pineconeTextKey || 'text'
|
||||
}
|
||||
|
||||
if (pineconeNamespace) obj.namespace = pineconeNamespace
|
||||
const pineconeStore = new PineconeStore(embeddings, obj)
|
||||
|
||||
try {
|
||||
if (recordManager) {
|
||||
const vectorStoreName = pineconeNamespace
|
||||
await recordManager.createSchema()
|
||||
;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
|
||||
const keys: string[] = await recordManager.listKeys({})
|
||||
|
||||
await pineconeStore.delete({ ids: keys })
|
||||
await recordManager.deleteKeys(keys)
|
||||
} else {
|
||||
const pineconeStore = new PineconeStore(embeddings, obj)
|
||||
await pineconeStore.delete({ ids })
|
||||
}
|
||||
} catch (e) {
|
||||
throw new Error(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -201,6 +201,58 @@ class Postgres_VectorStores implements INode {
|
||||
} catch (e) {
|
||||
throw new Error(e)
|
||||
}
|
||||
},
|
||||
async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise<void> {
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const user = getCredentialParam('user', credentialData, nodeData)
|
||||
const password = getCredentialParam('password', credentialData, nodeData)
|
||||
const _tableName = nodeData.inputs?.tableName as string
|
||||
const tableName = _tableName ? _tableName : 'documents'
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const additionalConfig = nodeData.inputs?.additionalConfig as string
|
||||
const recordManager = nodeData.inputs?.recordManager
|
||||
|
||||
let additionalConfiguration = {}
|
||||
if (additionalConfig) {
|
||||
try {
|
||||
additionalConfiguration = typeof additionalConfig === 'object' ? additionalConfig : JSON.parse(additionalConfig)
|
||||
} catch (exception) {
|
||||
throw new Error('Invalid JSON in the Additional Configuration: ' + exception)
|
||||
}
|
||||
}
|
||||
|
||||
const postgresConnectionOptions = {
|
||||
...additionalConfiguration,
|
||||
type: 'postgres',
|
||||
host: nodeData.inputs?.host as string,
|
||||
port: nodeData.inputs?.port as number,
|
||||
username: user,
|
||||
password: password,
|
||||
database: nodeData.inputs?.database as string
|
||||
}
|
||||
|
||||
const args = {
|
||||
postgresConnectionOptions: postgresConnectionOptions as DataSourceOptions,
|
||||
tableName: tableName
|
||||
}
|
||||
|
||||
const vectorStore = await TypeORMVectorStore.fromDataSource(embeddings, args)
|
||||
|
||||
try {
|
||||
if (recordManager) {
|
||||
const vectorStoreName = tableName
|
||||
await recordManager.createSchema()
|
||||
;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
|
||||
const keys: string[] = await recordManager.listKeys({})
|
||||
|
||||
await vectorStore.delete({ ids: keys })
|
||||
await recordManager.deleteKeys(keys)
|
||||
} else {
|
||||
await vectorStore.delete({ ids })
|
||||
}
|
||||
} catch (e) {
|
||||
throw new Error(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -291,6 +291,69 @@ class Qdrant_VectorStores implements INode {
|
||||
} catch (e) {
|
||||
throw new Error(e)
|
||||
}
|
||||
},
|
||||
async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise<void> {
|
||||
const qdrantServerUrl = nodeData.inputs?.qdrantServerUrl as string
|
||||
const collectionName = nodeData.inputs?.qdrantCollection as string
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const qdrantSimilarity = nodeData.inputs?.qdrantSimilarity
|
||||
const qdrantVectorDimension = nodeData.inputs?.qdrantVectorDimension
|
||||
const recordManager = nodeData.inputs?.recordManager
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const qdrantApiKey = getCredentialParam('qdrantApiKey', credentialData, nodeData)
|
||||
|
||||
const port = Qdrant_VectorStores.determinePortByUrl(qdrantServerUrl)
|
||||
|
||||
const client = new QdrantClient({
|
||||
url: qdrantServerUrl,
|
||||
apiKey: qdrantApiKey,
|
||||
port: port
|
||||
})
|
||||
|
||||
const dbConfig: QdrantLibArgs = {
|
||||
client,
|
||||
url: qdrantServerUrl,
|
||||
collectionName,
|
||||
collectionConfig: {
|
||||
vectors: {
|
||||
size: qdrantVectorDimension ? parseInt(qdrantVectorDimension, 10) : 1536,
|
||||
distance: qdrantSimilarity ?? 'Cosine'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const vectorStore = new QdrantVectorStore(embeddings, dbConfig)
|
||||
|
||||
vectorStore.delete = async (params: { ids: string[] }): Promise<void> => {
|
||||
const { ids } = params
|
||||
|
||||
if (ids?.length) {
|
||||
try {
|
||||
client.delete(collectionName, {
|
||||
points: ids
|
||||
})
|
||||
} catch (e) {
|
||||
console.error('Failed to delete')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
if (recordManager) {
|
||||
const vectorStoreName = collectionName
|
||||
await recordManager.createSchema()
|
||||
;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
|
||||
const keys: string[] = await recordManager.listKeys({})
|
||||
|
||||
await vectorStore.delete({ ids: keys })
|
||||
await recordManager.deleteKeys(keys)
|
||||
} else {
|
||||
await vectorStore.delete({ ids })
|
||||
}
|
||||
} catch (e) {
|
||||
throw new Error(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -171,6 +171,40 @@ class Supabase_VectorStores implements INode {
|
||||
} catch (e) {
|
||||
throw new Error(e)
|
||||
}
|
||||
},
|
||||
async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise<void> {
|
||||
const supabaseProjUrl = nodeData.inputs?.supabaseProjUrl as string
|
||||
const tableName = nodeData.inputs?.tableName as string
|
||||
const queryName = nodeData.inputs?.queryName as string
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const recordManager = nodeData.inputs?.recordManager
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const supabaseApiKey = getCredentialParam('supabaseApiKey', credentialData, nodeData)
|
||||
|
||||
const client = createClient(supabaseProjUrl, supabaseApiKey)
|
||||
|
||||
const supabaseStore = new SupabaseVectorStore(embeddings, {
|
||||
client,
|
||||
tableName: tableName,
|
||||
queryName: queryName
|
||||
})
|
||||
|
||||
try {
|
||||
if (recordManager) {
|
||||
const vectorStoreName = tableName + '_' + queryName
|
||||
await recordManager.createSchema()
|
||||
;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
|
||||
const keys: string[] = await recordManager.listKeys({})
|
||||
|
||||
await supabaseStore.delete({ ids: keys })
|
||||
await recordManager.deleteKeys(keys)
|
||||
} else {
|
||||
await supabaseStore.delete({ ids })
|
||||
}
|
||||
} catch (e) {
|
||||
throw new Error(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -145,6 +145,41 @@ class Upstash_VectorStores implements INode {
|
||||
} catch (e) {
|
||||
throw new Error(e)
|
||||
}
|
||||
},
|
||||
async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise<void> {
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const recordManager = nodeData.inputs?.recordManager
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const UPSTASH_VECTOR_REST_URL = getCredentialParam('UPSTASH_VECTOR_REST_URL', credentialData, nodeData)
|
||||
const UPSTASH_VECTOR_REST_TOKEN = getCredentialParam('UPSTASH_VECTOR_REST_TOKEN', credentialData, nodeData)
|
||||
|
||||
const upstashIndex = new UpstashIndex({
|
||||
url: UPSTASH_VECTOR_REST_URL,
|
||||
token: UPSTASH_VECTOR_REST_TOKEN
|
||||
})
|
||||
|
||||
const obj = {
|
||||
index: upstashIndex
|
||||
}
|
||||
|
||||
const upstashStore = new UpstashVectorStore(embeddings, obj)
|
||||
|
||||
try {
|
||||
if (recordManager) {
|
||||
const vectorStoreName = UPSTASH_VECTOR_REST_URL
|
||||
await recordManager.createSchema()
|
||||
;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
|
||||
const keys: string[] = await recordManager.listKeys({})
|
||||
|
||||
await upstashStore.delete({ ids: keys })
|
||||
await recordManager.deleteKeys(keys)
|
||||
} else {
|
||||
await upstashStore.delete({ ids })
|
||||
}
|
||||
} catch (e) {
|
||||
throw new Error(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -200,6 +200,53 @@ class Weaviate_VectorStores implements INode {
|
||||
} catch (e) {
|
||||
throw new Error(e)
|
||||
}
|
||||
},
|
||||
async delete(nodeData: INodeData, ids: string[], options: ICommonObject): Promise<void> {
|
||||
const weaviateScheme = nodeData.inputs?.weaviateScheme as string
|
||||
const weaviateHost = nodeData.inputs?.weaviateHost as string
|
||||
const weaviateIndex = nodeData.inputs?.weaviateIndex as string
|
||||
const weaviateTextKey = nodeData.inputs?.weaviateTextKey as string
|
||||
const weaviateMetadataKeys = nodeData.inputs?.weaviateMetadataKeys as string
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const recordManager = nodeData.inputs?.recordManager
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const weaviateApiKey = getCredentialParam('weaviateApiKey', credentialData, nodeData)
|
||||
|
||||
const clientConfig: any = {
|
||||
scheme: weaviateScheme,
|
||||
host: weaviateHost
|
||||
}
|
||||
if (weaviateApiKey) clientConfig.apiKey = new ApiKey(weaviateApiKey)
|
||||
|
||||
const client: WeaviateClient = weaviate.client(clientConfig)
|
||||
|
||||
const obj: WeaviateLibArgs = {
|
||||
//@ts-ignore
|
||||
client,
|
||||
indexName: weaviateIndex
|
||||
}
|
||||
|
||||
if (weaviateTextKey) obj.textKey = weaviateTextKey
|
||||
if (weaviateMetadataKeys) obj.metadataKeys = JSON.parse(weaviateMetadataKeys.replace(/\s/g, ''))
|
||||
|
||||
const weaviateStore = new WeaviateStore(embeddings, obj)
|
||||
|
||||
try {
|
||||
if (recordManager) {
|
||||
const vectorStoreName = weaviateTextKey ? weaviateIndex + '_' + weaviateTextKey : weaviateIndex
|
||||
await recordManager.createSchema()
|
||||
;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
|
||||
const keys: string[] = await recordManager.listKeys({})
|
||||
|
||||
await weaviateStore.delete({ ids: keys })
|
||||
await recordManager.deleteKeys(keys)
|
||||
} else {
|
||||
await weaviateStore.delete({ ids })
|
||||
}
|
||||
} catch (e) {
|
||||
throw new Error(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -130,7 +130,7 @@ export interface INode extends INodeProperties {
|
||||
vectorStoreMethods?: {
|
||||
upsert: (nodeData: INodeData, options?: ICommonObject) => Promise<IndexingResult | void>
|
||||
search: (nodeData: INodeData, options?: ICommonObject) => Promise<any>
|
||||
delete: (nodeData: INodeData, options?: ICommonObject) => Promise<void>
|
||||
delete: (nodeData: INodeData, ids: string[], options?: ICommonObject) => Promise<void>
|
||||
}
|
||||
init?(nodeData: INodeData, input: string, options?: ICommonObject): Promise<any>
|
||||
run?(nodeData: INodeData, input: string, options?: ICommonObject): Promise<string | ICommonObject>
|
||||
|
||||
@@ -5,7 +5,9 @@ export enum DocumentStoreStatus {
|
||||
SYNC = 'SYNC',
|
||||
SYNCING = 'SYNCING',
|
||||
STALE = 'STALE',
|
||||
NEW = 'NEW'
|
||||
NEW = 'NEW',
|
||||
UPSERTING = 'UPSERTING',
|
||||
UPSERTED = 'UPSERTED'
|
||||
}
|
||||
|
||||
export interface IDocumentStore {
|
||||
@@ -17,6 +19,9 @@ export interface IDocumentStore {
|
||||
updatedDate: Date
|
||||
createdDate: Date
|
||||
status: DocumentStoreStatus
|
||||
vectorStoreConfig: string | null // JSON string
|
||||
embeddingConfig: string | null // JSON string
|
||||
recordManagerConfig: string | null // JSON string
|
||||
}
|
||||
|
||||
export interface IDocumentStoreFileChunk {
|
||||
@@ -89,6 +94,9 @@ export class DocumentStoreDTO {
|
||||
totalChars: number
|
||||
chunkSize: number
|
||||
loaders: IDocumentStoreLoader[]
|
||||
vectorStoreConfig: any
|
||||
embeddingConfig: any
|
||||
recordManagerConfig: any
|
||||
|
||||
constructor() {}
|
||||
|
||||
@@ -109,6 +117,16 @@ export class DocumentStoreDTO {
|
||||
documentStoreDTO.whereUsed = []
|
||||
}
|
||||
|
||||
if (entity.vectorStoreConfig) {
|
||||
documentStoreDTO.vectorStoreConfig = JSON.parse(entity.vectorStoreConfig)
|
||||
}
|
||||
if (entity.embeddingConfig) {
|
||||
documentStoreDTO.embeddingConfig = JSON.parse(entity.embeddingConfig)
|
||||
}
|
||||
if (entity.recordManagerConfig) {
|
||||
documentStoreDTO.recordManagerConfig = JSON.parse(entity.recordManagerConfig)
|
||||
}
|
||||
|
||||
if (entity.loaders) {
|
||||
documentStoreDTO.loaders = JSON.parse(entity.loaders)
|
||||
documentStoreDTO.loaders.map((loader) => {
|
||||
|
||||
@@ -248,6 +248,100 @@ const getDocumentLoaders = async (req: Request, res: Response, next: NextFunctio
|
||||
}
|
||||
}
|
||||
|
||||
const insertIntoVectorStore = async (req: Request, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
if (typeof req.body === 'undefined') {
|
||||
throw new Error('Error: documentStoreController.insertIntoVectorStore - body not provided!')
|
||||
}
|
||||
const body = req.body
|
||||
const apiResponse = await documentStoreService.insertIntoVectorStore(body)
|
||||
return res.json(DocumentStoreDTO.fromEntity(apiResponse))
|
||||
} catch (error) {
|
||||
next(error)
|
||||
}
|
||||
}
|
||||
|
||||
const queryVectorStore = async (req: Request, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
if (typeof req.body === 'undefined') {
|
||||
throw new Error('Error: documentStoreController.queryVectorStore - body not provided!')
|
||||
}
|
||||
const body = req.body
|
||||
const apiResponse = await documentStoreService.queryVectorStore(body)
|
||||
return res.json(apiResponse)
|
||||
} catch (error) {
|
||||
next(error)
|
||||
}
|
||||
}
|
||||
|
||||
const deleteVectorStoreFromStore = async (req: Request, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
if (typeof req.params.storeId === 'undefined' || req.params.storeId === '') {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.PRECONDITION_FAILED,
|
||||
`Error: documentStoreController.deleteVectorStoreFromStore - storeId not provided!`
|
||||
)
|
||||
}
|
||||
const apiResponse = await documentStoreService.deleteVectorStoreFromStore(req.params.storeId)
|
||||
return res.json(apiResponse)
|
||||
} catch (error) {
|
||||
next(error)
|
||||
}
|
||||
}
|
||||
|
||||
const saveVectorStoreConfig = async (req: Request, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
if (typeof req.body === 'undefined') {
|
||||
throw new Error('Error: documentStoreController.saveVectorStoreConfig - body not provided!')
|
||||
}
|
||||
const body = req.body
|
||||
const apiResponse = await documentStoreService.saveVectorStoreConfig(body)
|
||||
return res.json(apiResponse)
|
||||
} catch (error) {
|
||||
next(error)
|
||||
}
|
||||
}
|
||||
|
||||
const updateVectorStoreConfigOnly = async (req: Request, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
if (typeof req.body === 'undefined') {
|
||||
throw new Error('Error: documentStoreController.updateVectorStoreConfigOnly - body not provided!')
|
||||
}
|
||||
const body = req.body
|
||||
const apiResponse = await documentStoreService.updateVectorStoreConfigOnly(body)
|
||||
return res.json(apiResponse)
|
||||
} catch (error) {
|
||||
next(error)
|
||||
}
|
||||
}
|
||||
|
||||
const getEmbeddingProviders = async (req: Request, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const apiResponse = await documentStoreService.getEmbeddingProviders()
|
||||
return res.json(apiResponse)
|
||||
} catch (error) {
|
||||
next(error)
|
||||
}
|
||||
}
|
||||
|
||||
const getVectorStoreProviders = async (req: Request, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const apiResponse = await documentStoreService.getVectorStoreProviders()
|
||||
return res.json(apiResponse)
|
||||
} catch (error) {
|
||||
next(error)
|
||||
}
|
||||
}
|
||||
|
||||
const getRecordManagerProviders = async (req: Request, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const apiResponse = await documentStoreService.getRecordManagerProviders()
|
||||
return res.json(apiResponse)
|
||||
} catch (error) {
|
||||
next(error)
|
||||
}
|
||||
}
|
||||
|
||||
export default {
|
||||
deleteDocumentStore,
|
||||
createDocumentStore,
|
||||
@@ -260,5 +354,13 @@ export default {
|
||||
previewFileChunks,
|
||||
getDocumentLoaders,
|
||||
deleteDocumentStoreFileChunk,
|
||||
editDocumentStoreFileChunk
|
||||
editDocumentStoreFileChunk,
|
||||
insertIntoVectorStore,
|
||||
getEmbeddingProviders,
|
||||
getVectorStoreProviders,
|
||||
getRecordManagerProviders,
|
||||
saveVectorStoreConfig,
|
||||
queryVectorStore,
|
||||
deleteVectorStoreFromStore,
|
||||
updateVectorStoreConfigOnly
|
||||
}
|
||||
|
||||
@@ -28,4 +28,13 @@ export class DocumentStore implements IDocumentStore {
|
||||
|
||||
@Column({ nullable: false, type: 'text' })
|
||||
status: DocumentStoreStatus
|
||||
|
||||
@Column({ nullable: true, type: 'text' })
|
||||
vectorStoreConfig: string | null
|
||||
|
||||
@Column({ nullable: true, type: 'text' })
|
||||
embeddingConfig: string | null
|
||||
|
||||
@Column({ nullable: true, type: 'text' })
|
||||
recordManagerConfig: string | null
|
||||
}
|
||||
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
import { MigrationInterface, QueryRunner } from 'typeorm'
|
||||
|
||||
export class AddVectorStoreConfigToDocStore1715861032479 implements MigrationInterface {
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
const columnExists = await queryRunner.hasColumn('document_store', 'vectorStoreConfig')
|
||||
if (!columnExists) {
|
||||
await queryRunner.query(`ALTER TABLE \`document_store\` ADD COLUMN \`vectorStoreConfig\` TEXT;`)
|
||||
await queryRunner.query(`ALTER TABLE \`document_store\` ADD COLUMN \`embeddingConfig\` TEXT;`)
|
||||
await queryRunner.query(`ALTER TABLE \`document_store\` ADD COLUMN \`recordManagerConfig\` TEXT;`)
|
||||
}
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`ALTER TABLE \`document_store\` DROP COLUMN \`vectorStoreConfig\`;`)
|
||||
await queryRunner.query(`ALTER TABLE \`document_store\` DROP COLUMN \`embeddingConfig\`;`)
|
||||
await queryRunner.query(`ALTER TABLE \`document_store\` DROP COLUMN \`recordManagerConfig\`;`)
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import { AddFeedback1707213626553 } from './1707213626553-AddFeedback'
|
||||
import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHistoryEntity'
|
||||
import { AddLead1710832127079 } from './1710832127079-AddLead'
|
||||
import { AddLeadToChatMessage1711538023578 } from './1711538023578-AddLeadToChatMessage'
|
||||
import { AddVectorStoreConfigToDocStore1715861032479 } from './1715861032479-AddVectorStoreConfigToDocStore'
|
||||
import { AddDocumentStore1711637331047 } from './1711637331047-AddDocumentStore'
|
||||
import { AddAgentReasoningToChatMessage1714679514451 } from './1714679514451-AddAgentReasoningToChatMessage'
|
||||
import { AddTypeToChatFlow1716300000000 } from './1716300000000-AddTypeToChatFlow'
|
||||
@@ -47,6 +48,7 @@ export const mysqlMigrations = [
|
||||
AddLeadToChatMessage1711538023578,
|
||||
AddAgentReasoningToChatMessage1714679514451,
|
||||
AddTypeToChatFlow1716300000000,
|
||||
AddVectorStoreConfigToDocStore1715861032479,
|
||||
AddApiKey1720230151480,
|
||||
AddActionToChatMessage1721078251523,
|
||||
LongTextColumn1722301395521
|
||||
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
import { MigrationInterface, QueryRunner } from 'typeorm'
|
||||
|
||||
export class AddVectorStoreConfigToDocStore1715861032479 implements MigrationInterface {
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`ALTER TABLE "document_store" ADD COLUMN IF NOT EXISTS "vectorStoreConfig" TEXT;`)
|
||||
await queryRunner.query(`ALTER TABLE "document_store" ADD COLUMN IF NOT EXISTS "embeddingConfig" TEXT;`)
|
||||
await queryRunner.query(`ALTER TABLE "document_store" ADD COLUMN IF NOT EXISTS "recordManagerConfig" TEXT;`)
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`ALTER TABLE "document_store" DROP COLUMN "vectorStoreConfig";`)
|
||||
await queryRunner.query(`ALTER TABLE "document_store" DROP COLUMN "embeddingConfig";`)
|
||||
await queryRunner.query(`ALTER TABLE "document_store" DROP COLUMN "recordManagerConfig";`)
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,7 @@ import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHi
|
||||
import { FieldTypes1710497452584 } from './1710497452584-FieldTypes'
|
||||
import { AddLead1710832137905 } from './1710832137905-AddLead'
|
||||
import { AddLeadToChatMessage1711538016098 } from './1711538016098-AddLeadToChatMessage'
|
||||
import { AddVectorStoreConfigToDocStore1715861032479 } from './1715861032479-AddVectorStoreConfigToDocStore'
|
||||
import { AddDocumentStore1711637331047 } from './1711637331047-AddDocumentStore'
|
||||
import { AddAgentReasoningToChatMessage1714679514451 } from './1714679514451-AddAgentReasoningToChatMessage'
|
||||
import { AddTypeToChatFlow1716300000000 } from './1716300000000-AddTypeToChatFlow'
|
||||
@@ -48,6 +49,7 @@ export const postgresMigrations = [
|
||||
AddLeadToChatMessage1711538016098,
|
||||
AddAgentReasoningToChatMessage1714679514451,
|
||||
AddTypeToChatFlow1716300000000,
|
||||
AddVectorStoreConfigToDocStore1715861032479,
|
||||
AddApiKey1720230151480,
|
||||
AddActionToChatMessage1721078251523
|
||||
]
|
||||
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
import { MigrationInterface, QueryRunner } from 'typeorm'
|
||||
|
||||
export class AddVectorStoreConfigToDocStore1715861032479 implements MigrationInterface {
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`ALTER TABLE "document_store" ADD COLUMN "vectorStoreConfig" TEXT;`)
|
||||
await queryRunner.query(`ALTER TABLE "document_store" ADD COLUMN "embeddingConfig" TEXT;`)
|
||||
await queryRunner.query(`ALTER TABLE "document_store" ADD COLUMN "recordManagerConfig" TEXT;`)
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`ALTER TABLE "document_store" DROP COLUMN "vectorStoreConfig";`)
|
||||
await queryRunner.query(`ALTER TABLE "document_store" DROP COLUMN "embeddingConfig";`)
|
||||
await queryRunner.query(`ALTER TABLE "document_store" DROP COLUMN "recordManagerConfig";`)
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import { AddFeedback1707213619308 } from './1707213619308-AddFeedback'
|
||||
import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHistoryEntity'
|
||||
import { AddLead1710832117612 } from './1710832117612-AddLead'
|
||||
import { AddLeadToChatMessage1711537986113 } from './1711537986113-AddLeadToChatMessage'
|
||||
import { AddVectorStoreConfigToDocStore1715861032479 } from './1715861032479-AddVectorStoreConfigToDocStore'
|
||||
import { AddDocumentStore1711637331047 } from './1711637331047-AddDocumentStore'
|
||||
import { AddAgentReasoningToChatMessage1714679514451 } from './1714679514451-AddAgentReasoningToChatMessage'
|
||||
import { AddTypeToChatFlow1716300000000 } from './1716300000000-AddTypeToChatFlow'
|
||||
@@ -46,6 +47,7 @@ export const sqliteMigrations = [
|
||||
AddLeadToChatMessage1711537986113,
|
||||
AddAgentReasoningToChatMessage1714679514451,
|
||||
AddTypeToChatFlow1716300000000,
|
||||
AddVectorStoreConfigToDocStore1715861032479,
|
||||
AddApiKey1720230151480,
|
||||
AddActionToChatMessage1721078251523
|
||||
]
|
||||
|
||||
@@ -16,7 +16,7 @@ router.delete('/store/:id', documentStoreController.deleteDocumentStore)
|
||||
|
||||
/** Component Nodes = Document Store - Loaders */
|
||||
// Get all loaders
|
||||
router.get('/loaders', documentStoreController.getDocumentLoaders)
|
||||
router.get('/components/loaders', documentStoreController.getDocumentLoaders)
|
||||
|
||||
// delete loader from document store
|
||||
router.delete('/loader/:id/:loaderId', documentStoreController.deleteLoaderFromDocumentStore)
|
||||
@@ -33,4 +33,22 @@ router.put('/chunks/:storeId/:loaderId/:chunkId', documentStoreController.editDo
|
||||
// Get all file chunks from the store
|
||||
router.get('/chunks/:storeId/:fileId/:pageNo', documentStoreController.getDocumentStoreFileChunks)
|
||||
|
||||
// add chunks to the selected vector store
|
||||
router.post('/vectorstore/insert', documentStoreController.insertIntoVectorStore)
|
||||
// save the selected vector store
|
||||
router.post('/vectorstore/save', documentStoreController.saveVectorStoreConfig)
|
||||
// delete data from the selected vector store
|
||||
router.delete('/vectorstore/:storeId', documentStoreController.deleteVectorStoreFromStore)
|
||||
// query the vector store
|
||||
router.post('/vectorstore/query', documentStoreController.queryVectorStore)
|
||||
// Get all embedding providers
|
||||
router.get('/components/embeddings', documentStoreController.getEmbeddingProviders)
|
||||
// Get all vector store providers
|
||||
router.get('/components/vectorstore', documentStoreController.getVectorStoreProviders)
|
||||
// Get all Record Manager providers
|
||||
router.get('/components/recordmanager', documentStoreController.getRecordManagerProviders)
|
||||
|
||||
// update the selected vector store from the playground
|
||||
router.post('/vectorstore/update', documentStoreController.updateVectorStoreConfigOnly)
|
||||
|
||||
export default router
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
|
||||
import { DocumentStore } from '../../database/entities/DocumentStore'
|
||||
// @ts-ignore
|
||||
import {
|
||||
addSingleFileToStorage,
|
||||
getFileFromStorage,
|
||||
@@ -10,22 +9,29 @@ import {
|
||||
removeSpecificFileFromStorage
|
||||
} from 'flowise-components'
|
||||
import {
|
||||
chatType,
|
||||
DocumentStoreStatus,
|
||||
IDocumentStoreFileChunkPagedResponse,
|
||||
IDocumentStoreLoader,
|
||||
IDocumentStoreLoaderFile,
|
||||
IDocumentStoreLoaderForPreview,
|
||||
IDocumentStoreWhereUsed
|
||||
IDocumentStoreWhereUsed,
|
||||
INodeData
|
||||
} from '../../Interface'
|
||||
import { DocumentStoreFileChunk } from '../../database/entities/DocumentStoreFileChunk'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
import { databaseEntities } from '../../utils'
|
||||
import { databaseEntities, getAppVersion, saveUpsertFlowData } from '../../utils'
|
||||
import logger from '../../utils/logger'
|
||||
import nodesService from '../nodes'
|
||||
import { InternalFlowiseError } from '../../errors/internalFlowiseError'
|
||||
import { StatusCodes } from 'http-status-codes'
|
||||
import { getErrorMessage } from '../../errors/utils'
|
||||
import { ChatFlow } from '../../database/entities/ChatFlow'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { App } from '../../index'
|
||||
import { UpsertHistory } from '../../database/entities/UpsertHistory'
|
||||
import { cloneDeep, omit } from 'lodash'
|
||||
import telemetryService from '../telemetry'
|
||||
|
||||
const DOCUMENT_STORE_BASE_FOLDER = 'docustore'
|
||||
|
||||
@@ -234,8 +240,16 @@ const deleteDocumentStore = async (storeId: string) => {
|
||||
const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({
|
||||
id: storeId
|
||||
})
|
||||
if (!entity) throw new Error(`Document store ${storeId} not found`)
|
||||
if (!entity) {
|
||||
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document store ${storeId} not found`)
|
||||
}
|
||||
await removeFilesFromStorage(DOCUMENT_STORE_BASE_FOLDER, entity.id)
|
||||
|
||||
// delete upsert history
|
||||
await appServer.AppDataSource.getRepository(UpsertHistory).delete({
|
||||
chatflowid: storeId
|
||||
})
|
||||
|
||||
// now delete the store
|
||||
const tbd = await appServer.AppDataSource.getRepository(DocumentStore).delete({
|
||||
id: storeId
|
||||
@@ -285,6 +299,83 @@ const deleteDocumentStoreFileChunk = async (storeId: string, docId: string, chun
|
||||
}
|
||||
}
|
||||
|
||||
const deleteVectorStoreFromStore = async (storeId: string) => {
|
||||
try {
|
||||
const appServer = getRunningExpressApp()
|
||||
const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({
|
||||
id: storeId
|
||||
})
|
||||
if (!entity) {
|
||||
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document store ${storeId} not found`)
|
||||
}
|
||||
|
||||
if (!entity.embeddingConfig) {
|
||||
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Embedding for Document store ${storeId} not found`)
|
||||
}
|
||||
|
||||
if (!entity.vectorStoreConfig) {
|
||||
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Vector Store for Document store ${storeId} not found`)
|
||||
}
|
||||
|
||||
if (!entity.recordManagerConfig) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.NOT_FOUND,
|
||||
`Record Manager for Document Store ${storeId} is needed to delete data from Vector Store`
|
||||
)
|
||||
}
|
||||
|
||||
const options: ICommonObject = {
|
||||
chatflowid: storeId,
|
||||
appDataSource: appServer.AppDataSource,
|
||||
databaseEntities,
|
||||
logger
|
||||
}
|
||||
|
||||
// Get Record Manager Instance
|
||||
const recordManagerConfig = JSON.parse(entity.recordManagerConfig)
|
||||
const recordManagerObj = await _createRecordManagerObject(
|
||||
appServer,
|
||||
{ recordManagerName: recordManagerConfig.name, recordManagerConfig: recordManagerConfig.config },
|
||||
options
|
||||
)
|
||||
|
||||
// Get Embeddings Instance
|
||||
const embeddingConfig = JSON.parse(entity.embeddingConfig)
|
||||
const embeddingObj = await _createEmbeddingsObject(
|
||||
appServer,
|
||||
{ embeddingName: embeddingConfig.name, embeddingConfig: embeddingConfig.config },
|
||||
options
|
||||
)
|
||||
|
||||
// Get Vector Store Node Data
|
||||
const vectorStoreConfig = JSON.parse(entity.vectorStoreConfig)
|
||||
const vStoreNodeData = _createVectorStoreNodeData(
|
||||
appServer,
|
||||
{ vectorStoreName: vectorStoreConfig.name, vectorStoreConfig: vectorStoreConfig.config },
|
||||
embeddingObj,
|
||||
recordManagerObj
|
||||
)
|
||||
|
||||
// Get Vector Store Instance
|
||||
const vectorStoreObj = await _createVectorStoreObject(
|
||||
appServer,
|
||||
{ vectorStoreName: vectorStoreConfig.name, vectorStoreConfig: vectorStoreConfig.config },
|
||||
vStoreNodeData
|
||||
)
|
||||
const idsToDelete: string[] = [] // empty ids because we get it dynamically from the record manager
|
||||
|
||||
// Call the delete method of the vector store
|
||||
if (vectorStoreObj.vectorStoreMethods.delete) {
|
||||
await vectorStoreObj.vectorStoreMethods.delete(vStoreNodeData, idsToDelete, options)
|
||||
}
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: documentStoreServices.deleteVectorStoreFromStore - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const editDocumentStoreFileChunk = async (storeId: string, docId: string, chunkId: string, content: string, metadata: ICommonObject) => {
|
||||
try {
|
||||
const appServer = getRunningExpressApp()
|
||||
@@ -700,6 +791,417 @@ const updateDocumentStoreUsage = async (chatId: string, storeId: string | undefi
|
||||
}
|
||||
}
|
||||
|
||||
const updateVectorStoreConfigOnly = async (data: ICommonObject) => {
|
||||
try {
|
||||
const appServer = getRunningExpressApp()
|
||||
const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({
|
||||
id: data.storeId
|
||||
})
|
||||
if (!entity) {
|
||||
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document store ${data.storeId} not found`)
|
||||
}
|
||||
|
||||
if (data.vectorStoreName) {
|
||||
entity.vectorStoreConfig = JSON.stringify({
|
||||
config: data.vectorStoreConfig,
|
||||
name: data.vectorStoreName
|
||||
})
|
||||
|
||||
const updatedEntity = await appServer.AppDataSource.getRepository(DocumentStore).save(entity)
|
||||
return updatedEntity
|
||||
}
|
||||
return {}
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: documentStoreServices.updateVectorStoreConfig - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
const saveVectorStoreConfig = async (data: ICommonObject) => {
|
||||
try {
|
||||
const appServer = getRunningExpressApp()
|
||||
const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({
|
||||
id: data.storeId
|
||||
})
|
||||
if (!entity) {
|
||||
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document store ${data.storeId} not found`)
|
||||
}
|
||||
|
||||
if (data.embeddingName) {
|
||||
entity.embeddingConfig = JSON.stringify({
|
||||
config: data.embeddingConfig,
|
||||
name: data.embeddingName
|
||||
})
|
||||
} else if (!data.embeddingName && !data.embeddingConfig) {
|
||||
entity.embeddingConfig = null
|
||||
}
|
||||
|
||||
if (data.vectorStoreName) {
|
||||
entity.vectorStoreConfig = JSON.stringify({
|
||||
config: data.vectorStoreConfig,
|
||||
name: data.vectorStoreName
|
||||
})
|
||||
} else if (!data.vectorStoreName && !data.vectorStoreConfig) {
|
||||
entity.vectorStoreConfig = null
|
||||
}
|
||||
|
||||
if (data.recordManagerName) {
|
||||
entity.recordManagerConfig = JSON.stringify({
|
||||
config: data.recordManagerConfig,
|
||||
name: data.recordManagerName
|
||||
})
|
||||
} else if (!data.recordManagerName && !data.recordManagerConfig) {
|
||||
entity.recordManagerConfig = null
|
||||
}
|
||||
|
||||
if (entity.status !== DocumentStoreStatus.UPSERTED && (data.vectorStoreName || data.recordManagerName || data.embeddingName)) {
|
||||
// if the store is not already in sync, mark it as sync
|
||||
// this also means that the store is not yet sync'ed to vector store
|
||||
entity.status = DocumentStoreStatus.SYNC
|
||||
}
|
||||
await appServer.AppDataSource.getRepository(DocumentStore).save(entity)
|
||||
return entity
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: documentStoreServices.saveVectorStoreConfig - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const insertIntoVectorStore = async (data: ICommonObject) => {
|
||||
try {
|
||||
const appServer = getRunningExpressApp()
|
||||
const entity = await saveVectorStoreConfig(data)
|
||||
entity.status = DocumentStoreStatus.UPSERTING
|
||||
await appServer.AppDataSource.getRepository(DocumentStore).save(entity)
|
||||
|
||||
// TODO: to be moved into a worker thread...
|
||||
const indexResult = await _insertIntoVectorStoreWorkerThread(data)
|
||||
return indexResult
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: documentStoreServices.insertIntoVectorStore - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const _insertIntoVectorStoreWorkerThread = async (data: ICommonObject) => {
|
||||
try {
|
||||
const appServer = getRunningExpressApp()
|
||||
const entity = await saveVectorStoreConfig(data)
|
||||
let upsertHistory: Record<string, any> = {}
|
||||
const chatflowid = data.storeId // fake chatflowid because this is not tied to any chatflow
|
||||
|
||||
const options: ICommonObject = {
|
||||
chatflowid,
|
||||
appDataSource: appServer.AppDataSource,
|
||||
databaseEntities,
|
||||
logger
|
||||
}
|
||||
|
||||
let recordManagerObj = undefined
|
||||
|
||||
// Get Record Manager Instance
|
||||
if (data.recordManagerName && data.recordManagerConfig) {
|
||||
recordManagerObj = await _createRecordManagerObject(appServer, data, options, upsertHistory)
|
||||
}
|
||||
|
||||
// Get Embeddings Instance
|
||||
const embeddingObj = await _createEmbeddingsObject(appServer, data, options, upsertHistory)
|
||||
|
||||
// Get Vector Store Node Data
|
||||
const vStoreNodeData = _createVectorStoreNodeData(appServer, data, embeddingObj, recordManagerObj)
|
||||
// Prepare docs for upserting
|
||||
const chunks = await appServer.AppDataSource.getRepository(DocumentStoreFileChunk).find({
|
||||
where: {
|
||||
storeId: data.storeId
|
||||
}
|
||||
})
|
||||
const docs: Document[] = chunks.map((chunk: DocumentStoreFileChunk) => {
|
||||
return new Document({
|
||||
pageContent: chunk.pageContent,
|
||||
metadata: JSON.parse(chunk.metadata)
|
||||
})
|
||||
})
|
||||
vStoreNodeData.inputs.document = docs
|
||||
|
||||
// Get Vector Store Instance
|
||||
const vectorStoreObj = await _createVectorStoreObject(appServer, data, vStoreNodeData, upsertHistory)
|
||||
const indexResult = await vectorStoreObj.vectorStoreMethods.upsert(vStoreNodeData, options)
|
||||
|
||||
// Save to DB
|
||||
if (indexResult) {
|
||||
const result = cloneDeep(upsertHistory)
|
||||
result['flowData'] = JSON.stringify(result['flowData'])
|
||||
result['result'] = JSON.stringify(omit(indexResult, ['totalKeys', 'addedDocs']))
|
||||
result.chatflowid = chatflowid
|
||||
const newUpsertHistory = new UpsertHistory()
|
||||
Object.assign(newUpsertHistory, result)
|
||||
const upsertHistoryItem = appServer.AppDataSource.getRepository(UpsertHistory).create(newUpsertHistory)
|
||||
await appServer.AppDataSource.getRepository(UpsertHistory).save(upsertHistoryItem)
|
||||
}
|
||||
|
||||
await telemetryService.createEvent({
|
||||
name: `vector_upserted`,
|
||||
data: {
|
||||
version: await getAppVersion(),
|
||||
chatlowId: chatflowid,
|
||||
type: chatType.INTERNAL,
|
||||
flowGraph: omit(indexResult['result'], ['totalKeys', 'addedDocs'])
|
||||
}
|
||||
})
|
||||
|
||||
entity.status = DocumentStoreStatus.UPSERTED
|
||||
await appServer.AppDataSource.getRepository(DocumentStore).save(entity)
|
||||
|
||||
return indexResult ?? { result: 'Successfully Upserted' }
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: documentStoreServices._insertIntoVectorStoreWorkerThread - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Get all component nodes - Embeddings
|
||||
const getEmbeddingProviders = async () => {
|
||||
try {
|
||||
const dbResponse = await nodesService.getAllNodesForCategory('Embeddings')
|
||||
return dbResponse.filter((node) => !node.tags?.includes('LlamaIndex'))
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: documentStoreServices.getEmbeddingProviders - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Get all component nodes - Vector Stores
|
||||
const getVectorStoreProviders = async () => {
|
||||
try {
|
||||
const dbResponse = await nodesService.getAllNodesForCategory('Vector Stores')
|
||||
return dbResponse.filter((node) => !node.tags?.includes('LlamaIndex') && node.name !== 'documentStoreVS')
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: documentStoreServices.getVectorStoreProviders - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
// Get all component nodes - Vector Stores
|
||||
const getRecordManagerProviders = async () => {
|
||||
try {
|
||||
const dbResponse = await nodesService.getAllNodesForCategory('Record Manager')
|
||||
return dbResponse.filter((node) => !node.tags?.includes('LlamaIndex'))
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: documentStoreServices.getRecordManagerProviders - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const queryVectorStore = async (data: ICommonObject) => {
|
||||
try {
|
||||
const appServer = getRunningExpressApp()
|
||||
const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({
|
||||
id: data.storeId
|
||||
})
|
||||
if (!entity) {
|
||||
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Document store ${data.storeId} not found`)
|
||||
}
|
||||
const options: ICommonObject = {
|
||||
chatflowid: uuidv4(),
|
||||
appDataSource: appServer.AppDataSource,
|
||||
databaseEntities,
|
||||
logger
|
||||
}
|
||||
|
||||
if (!entity.embeddingConfig) {
|
||||
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Embedding for ${data.storeId} is not configured`)
|
||||
}
|
||||
|
||||
if (!entity.vectorStoreConfig) {
|
||||
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Vector Store for ${data.storeId} is not configured`)
|
||||
}
|
||||
|
||||
const embeddingConfig = JSON.parse(entity.embeddingConfig)
|
||||
data.embeddingName = embeddingConfig.name
|
||||
data.embeddingConfig = embeddingConfig.config
|
||||
let embeddingObj = await _createEmbeddingsObject(appServer, data, options)
|
||||
|
||||
const vsConfig = JSON.parse(entity.vectorStoreConfig)
|
||||
data.vectorStoreName = vsConfig.name
|
||||
data.vectorStoreConfig = vsConfig.config
|
||||
if (data.inputs) {
|
||||
data.vectorStoreConfig = { ...vsConfig.config, ...data.inputs }
|
||||
}
|
||||
|
||||
const vStoreNodeData = _createVectorStoreNodeData(appServer, data, embeddingObj, undefined)
|
||||
|
||||
// Get Vector Store Instance
|
||||
const vectorStoreObj = await _createVectorStoreObject(appServer, data, vStoreNodeData)
|
||||
const retriever = await vectorStoreObj.init(vStoreNodeData, '', options)
|
||||
if (!retriever) {
|
||||
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Failed to create retriever`)
|
||||
}
|
||||
const startMillis = Date.now()
|
||||
const results = await retriever.invoke(data.query, undefined)
|
||||
if (!results) {
|
||||
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Failed to retrieve results`)
|
||||
}
|
||||
const endMillis = Date.now()
|
||||
const timeTaken = endMillis - startMillis
|
||||
const docs: any = results.map((result: IDocument) => {
|
||||
return {
|
||||
pageContent: result.pageContent,
|
||||
metadata: result.metadata,
|
||||
id: uuidv4()
|
||||
}
|
||||
})
|
||||
// query our document store chunk with the storeId and pageContent
|
||||
for (const doc of docs) {
|
||||
const documentStoreChunk = await appServer.AppDataSource.getRepository(DocumentStoreFileChunk).findOneBy({
|
||||
storeId: data.storeId,
|
||||
pageContent: doc.pageContent
|
||||
})
|
||||
if (documentStoreChunk) {
|
||||
doc.id = documentStoreChunk.id
|
||||
doc.chunkNo = documentStoreChunk.chunkNo
|
||||
} else {
|
||||
// this should not happen, only possible if the vector store has more content
|
||||
// than our document store
|
||||
doc.id = uuidv4()
|
||||
doc.chunkNo = -1
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
timeTaken: timeTaken,
|
||||
docs: docs
|
||||
}
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: documentStoreServices.queryVectorStore - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const _createEmbeddingsObject = async (
|
||||
appServer: App,
|
||||
data: ICommonObject,
|
||||
options: ICommonObject,
|
||||
upsertHistory?: Record<string, any>
|
||||
): Promise<any> => {
|
||||
// prepare embedding node data
|
||||
const embeddingComponent = appServer.nodesPool.componentNodes[data.embeddingName]
|
||||
const embeddingNodeData: any = {
|
||||
inputs: { ...data.embeddingConfig },
|
||||
outputs: { output: 'document' },
|
||||
id: `${embeddingComponent.name}_0`,
|
||||
label: embeddingComponent.label,
|
||||
name: embeddingComponent.name,
|
||||
category: embeddingComponent.category,
|
||||
inputParams: embeddingComponent.inputs || []
|
||||
}
|
||||
if (data.embeddingConfig.credential) {
|
||||
embeddingNodeData.credential = data.embeddingConfig.credential
|
||||
}
|
||||
|
||||
// save to upsert history
|
||||
if (upsertHistory) upsertHistory['flowData'] = saveUpsertFlowData(embeddingNodeData, upsertHistory)
|
||||
|
||||
// init embedding object
|
||||
const embeddingNodeInstanceFilePath = embeddingComponent.filePath as string
|
||||
const embeddingNodeModule = await import(embeddingNodeInstanceFilePath)
|
||||
const embeddingNodeInstance = new embeddingNodeModule.nodeClass()
|
||||
const embeddingObj = await embeddingNodeInstance.init(embeddingNodeData, '', options)
|
||||
if (!embeddingObj) {
|
||||
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Failed to create EmbeddingObj`)
|
||||
}
|
||||
return embeddingObj
|
||||
}
|
||||
|
||||
const _createRecordManagerObject = async (
|
||||
appServer: App,
|
||||
data: ICommonObject,
|
||||
options: ICommonObject,
|
||||
upsertHistory?: Record<string, any>
|
||||
) => {
|
||||
// prepare record manager node data
|
||||
const recordManagerComponent = appServer.nodesPool.componentNodes[data.recordManagerName]
|
||||
const rmNodeData: any = {
|
||||
inputs: { ...data.recordManagerConfig },
|
||||
id: `${recordManagerComponent.name}_0`,
|
||||
inputParams: recordManagerComponent.inputs,
|
||||
label: recordManagerComponent.label,
|
||||
name: recordManagerComponent.name,
|
||||
category: recordManagerComponent.category
|
||||
}
|
||||
if (data.recordManagerConfig.credential) {
|
||||
rmNodeData.credential = data.recordManagerConfig.credential
|
||||
}
|
||||
|
||||
// save to upsert history
|
||||
if (upsertHistory) upsertHistory['flowData'] = saveUpsertFlowData(rmNodeData, upsertHistory)
|
||||
|
||||
// init record manager object
|
||||
const rmNodeInstanceFilePath = recordManagerComponent.filePath as string
|
||||
const rmNodeModule = await import(rmNodeInstanceFilePath)
|
||||
const rmNodeInstance = new rmNodeModule.nodeClass()
|
||||
const recordManagerObj = await rmNodeInstance.init(rmNodeData, '', options)
|
||||
if (!recordManagerObj) {
|
||||
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Failed to create RecordManager obj`)
|
||||
}
|
||||
return recordManagerObj
|
||||
}
|
||||
|
||||
const _createVectorStoreNodeData = (appServer: App, data: ICommonObject, embeddingObj: any, recordManagerObj?: any) => {
|
||||
const vectorStoreComponent = appServer.nodesPool.componentNodes[data.vectorStoreName]
|
||||
const vStoreNodeData: any = {
|
||||
id: `${vectorStoreComponent.name}_0`,
|
||||
inputs: { ...data.vectorStoreConfig },
|
||||
outputs: { output: 'retriever' },
|
||||
label: vectorStoreComponent.label,
|
||||
name: vectorStoreComponent.name,
|
||||
category: vectorStoreComponent.category
|
||||
}
|
||||
if (data.vectorStoreConfig.credential) {
|
||||
vStoreNodeData.credential = data.vectorStoreConfig.credential
|
||||
}
|
||||
|
||||
if (embeddingObj) {
|
||||
vStoreNodeData.inputs.embeddings = embeddingObj
|
||||
}
|
||||
|
||||
if (recordManagerObj) {
|
||||
vStoreNodeData.inputs.recordManager = recordManagerObj
|
||||
}
|
||||
|
||||
// Get all input params except the ones that are anchor points to avoid JSON stringify circular error
|
||||
const filterInputParams = ['document', 'embeddings', 'recordManager']
|
||||
const inputParams = vectorStoreComponent.inputs?.filter((input) => !filterInputParams.includes(input.name))
|
||||
vStoreNodeData.inputParams = inputParams
|
||||
return vStoreNodeData
|
||||
}
|
||||
|
||||
const _createVectorStoreObject = async (
|
||||
appServer: App,
|
||||
data: ICommonObject,
|
||||
vStoreNodeData: INodeData,
|
||||
upsertHistory?: Record<string, any>
|
||||
) => {
|
||||
const vStoreNodeInstanceFilePath = appServer.nodesPool.componentNodes[data.vectorStoreName].filePath as string
|
||||
const vStoreNodeModule = await import(vStoreNodeInstanceFilePath)
|
||||
const vStoreNodeInstance = new vStoreNodeModule.nodeClass()
|
||||
if (upsertHistory) upsertHistory['flowData'] = saveUpsertFlowData(vStoreNodeData, upsertHistory)
|
||||
return vStoreNodeInstance
|
||||
}
|
||||
|
||||
export default {
|
||||
updateDocumentStoreUsage,
|
||||
deleteDocumentStore,
|
||||
@@ -714,5 +1216,13 @@ export default {
|
||||
processAndSaveChunks,
|
||||
deleteDocumentStoreFileChunk,
|
||||
editDocumentStoreFileChunk,
|
||||
getDocumentLoaders
|
||||
getDocumentLoaders,
|
||||
insertIntoVectorStore,
|
||||
getEmbeddingProviders,
|
||||
getVectorStoreProviders,
|
||||
getRecordManagerProviders,
|
||||
saveVectorStoreConfig,
|
||||
queryVectorStore,
|
||||
deleteVectorStoreFromStore,
|
||||
updateVectorStoreConfigOnly
|
||||
}
|
||||
|
||||
@@ -547,7 +547,8 @@ export const buildFlow = async ({
|
||||
uploads,
|
||||
baseURL,
|
||||
socketIO,
|
||||
socketIOClientId
|
||||
socketIOClientId,
|
||||
componentNodes: componentNodes as ICommonObject
|
||||
})
|
||||
|
||||
// Save dynamic variables
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import client from './client'
|
||||
|
||||
const getAllDocumentStores = () => client.get('/document-store/stores')
|
||||
const getDocumentLoaders = () => client.get('/document-store/loaders')
|
||||
const getDocumentLoaders = () => client.get('/document-store/components/loaders')
|
||||
const getSpecificDocumentStore = (id) => client.get(`/document-store/store/${id}`)
|
||||
const createDocumentStore = (body) => client.post(`/document-store/store`, body)
|
||||
const updateDocumentStore = (id, body) => client.put(`/document-store/store/${id}`, body)
|
||||
@@ -16,6 +16,15 @@ const getFileChunks = (storeId, fileId, pageNo) => client.get(`/document-store/c
|
||||
const previewChunks = (body) => client.post('/document-store/loader/preview', body)
|
||||
const processChunks = (body) => client.post(`/document-store/loader/process`, body)
|
||||
|
||||
const insertIntoVectorStore = (body) => client.post(`/document-store/vectorstore/insert`, body)
|
||||
const saveVectorStoreConfig = (body) => client.post(`/document-store/vectorstore/save`, body)
|
||||
const updateVectorStoreConfig = (body) => client.post(`/document-store/vectorstore/update`, body)
|
||||
const deleteVectorStoreDataFromStore = (storeId) => client.delete(`/document-store/vectorstore/${storeId}`)
|
||||
const queryVectorStore = (body) => client.post(`/document-store/vectorstore/query`, body)
|
||||
const getVectorStoreProviders = () => client.get('/document-store/components/vectorstore')
|
||||
const getEmbeddingProviders = () => client.get('/document-store/components/embeddings')
|
||||
const getRecordManagerProviders = () => client.get('/document-store/components/recordmanager')
|
||||
|
||||
export default {
|
||||
getAllDocumentStores,
|
||||
getSpecificDocumentStore,
|
||||
@@ -28,5 +37,13 @@ export default {
|
||||
getDocumentLoaders,
|
||||
deleteChunkFromStore,
|
||||
editChunkFromStore,
|
||||
deleteDocumentStore
|
||||
deleteDocumentStore,
|
||||
insertIntoVectorStore,
|
||||
getVectorStoreProviders,
|
||||
getEmbeddingProviders,
|
||||
getRecordManagerProviders,
|
||||
saveVectorStoreConfig,
|
||||
queryVectorStore,
|
||||
deleteVectorStoreDataFromStore,
|
||||
updateVectorStoreConfig
|
||||
}
|
||||
|
||||
@@ -33,6 +33,8 @@ const Documents = Loadable(lazy(() => import('@/views/docstore')))
|
||||
const DocumentStoreDetail = Loadable(lazy(() => import('@/views/docstore/DocumentStoreDetail')))
|
||||
const ShowStoredChunks = Loadable(lazy(() => import('@/views/docstore/ShowStoredChunks')))
|
||||
const LoaderConfigPreviewChunks = Loadable(lazy(() => import('@/views/docstore/LoaderConfigPreviewChunks')))
|
||||
const VectorStoreConfigure = Loadable(lazy(() => import('@/views/docstore/VectorStoreConfigure')))
|
||||
const VectorStoreQuery = Loadable(lazy(() => import('@/views/docstore/VectorStoreQuery')))
|
||||
|
||||
// ==============================|| MAIN ROUTING ||============================== //
|
||||
|
||||
@@ -91,6 +93,14 @@ const MainRoutes = {
|
||||
{
|
||||
path: '/document-stores/:id/:name',
|
||||
element: <LoaderConfigPreviewChunks />
|
||||
},
|
||||
{
|
||||
path: '/document-stores/vector/:id',
|
||||
element: <VectorStoreConfigure />
|
||||
},
|
||||
{
|
||||
path: '/document-stores/query/:id',
|
||||
element: <VectorStoreQuery />
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
import { useState, useEffect } from 'react'
|
||||
import { createPortal } from 'react-dom'
|
||||
import { useSelector, useDispatch } from 'react-redux'
|
||||
import PropTypes from 'prop-types'
|
||||
import { List, ListItemButton, Dialog, DialogContent, DialogTitle, Box, OutlinedInput, InputAdornment, Typography } from '@mui/material'
|
||||
import { useTheme } from '@mui/material/styles'
|
||||
import { IconSearch, IconX } from '@tabler/icons-react'
|
||||
|
||||
// API
|
||||
|
||||
// const
|
||||
import { baseURL } from '@/store/constant'
|
||||
import { HIDE_CANVAS_DIALOG, SHOW_CANVAS_DIALOG } from '@/store/actions'
|
||||
import useApi from '@/hooks/useApi'
|
||||
|
||||
const ComponentsListDialog = ({ show, dialogProps, onCancel, apiCall, onSelected }) => {
|
||||
const portalElement = document.getElementById('portal')
|
||||
const customization = useSelector((state) => state.customization)
|
||||
const dispatch = useDispatch()
|
||||
const theme = useTheme()
|
||||
const [searchValue, setSearchValue] = useState('')
|
||||
const [provider, setProvider] = useState([])
|
||||
|
||||
const getProvidersApi = useApi(apiCall)
|
||||
|
||||
const onSearchChange = (val) => {
|
||||
setSearchValue(val)
|
||||
}
|
||||
|
||||
function filterFlows(data) {
|
||||
return data?.name?.toLowerCase().indexOf(searchValue.toLowerCase()) > -1
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
// if (dialogProps.embeddingsProvider) {
|
||||
// setProvider(dialogProps.provider)
|
||||
// }
|
||||
}, [dialogProps])
|
||||
|
||||
useEffect(() => {
|
||||
getProvidersApi.request()
|
||||
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
if (getProvidersApi.data) {
|
||||
setProvider(getProvidersApi.data)
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [getProvidersApi.data])
|
||||
|
||||
useEffect(() => {
|
||||
if (show) dispatch({ type: SHOW_CANVAS_DIALOG })
|
||||
else dispatch({ type: HIDE_CANVAS_DIALOG })
|
||||
return () => dispatch({ type: HIDE_CANVAS_DIALOG })
|
||||
}, [show, dispatch])
|
||||
|
||||
const component = show ? (
|
||||
<Dialog
|
||||
fullWidth
|
||||
maxWidth='md'
|
||||
open={show}
|
||||
onClose={onCancel}
|
||||
aria-labelledby='alert-dialog-title'
|
||||
aria-describedby='alert-dialog-description'
|
||||
>
|
||||
<DialogTitle sx={{ fontSize: '1rem', p: 3, pb: 0 }} id='alert-dialog-title'>
|
||||
{dialogProps.title}
|
||||
</DialogTitle>
|
||||
<DialogContent sx={{ display: 'flex', flexDirection: 'column', gap: 2, maxHeight: '75vh', position: 'relative', px: 3, pb: 3 }}>
|
||||
<Box
|
||||
sx={{
|
||||
backgroundColor: customization.isDarkMode ? theme.palette.background.darkPaper : theme.palette.background.paper,
|
||||
pt: 2,
|
||||
position: 'sticky',
|
||||
top: 0,
|
||||
zIndex: 10
|
||||
}}
|
||||
>
|
||||
<OutlinedInput
|
||||
sx={{ width: '100%', pr: 2, pl: 2, position: 'sticky' }}
|
||||
id='input-search-credential'
|
||||
value={searchValue}
|
||||
onChange={(e) => onSearchChange(e.target.value)}
|
||||
placeholder='Search'
|
||||
startAdornment={
|
||||
<InputAdornment position='start'>
|
||||
<IconSearch stroke={1.5} size='1rem' color={theme.palette.grey[500]} />
|
||||
</InputAdornment>
|
||||
}
|
||||
endAdornment={
|
||||
<InputAdornment
|
||||
position='end'
|
||||
sx={{
|
||||
cursor: 'pointer',
|
||||
color: theme.palette.grey[500],
|
||||
'&:hover': {
|
||||
color: theme.palette.grey[900]
|
||||
}
|
||||
}}
|
||||
title='Clear Search'
|
||||
>
|
||||
<IconX
|
||||
stroke={1.5}
|
||||
size='1rem'
|
||||
onClick={() => onSearchChange('')}
|
||||
style={{
|
||||
cursor: 'pointer'
|
||||
}}
|
||||
/>
|
||||
</InputAdornment>
|
||||
}
|
||||
aria-describedby='search-helper-text'
|
||||
inputProps={{
|
||||
'aria-label': 'weight'
|
||||
}}
|
||||
/>
|
||||
</Box>
|
||||
<List
|
||||
sx={{
|
||||
width: '100%',
|
||||
display: 'grid',
|
||||
gridTemplateColumns: 'repeat(3, 1fr)',
|
||||
gap: 2,
|
||||
py: 0,
|
||||
zIndex: 9,
|
||||
borderRadius: '10px',
|
||||
[theme.breakpoints.down('md')]: {
|
||||
maxWidth: 370
|
||||
}
|
||||
}}
|
||||
>
|
||||
{[...provider].filter(filterFlows).map((loader) => (
|
||||
<ListItemButton
|
||||
alignItems='center'
|
||||
key={loader.name}
|
||||
onClick={() => onSelected(loader)}
|
||||
sx={{
|
||||
border: 1,
|
||||
borderColor: theme.palette.grey[900] + 25,
|
||||
borderRadius: 2,
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'start',
|
||||
textAlign: 'left',
|
||||
gap: 1,
|
||||
p: 2
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
width: 50,
|
||||
height: 50,
|
||||
borderRadius: '50%',
|
||||
backgroundColor: 'white'
|
||||
}}
|
||||
>
|
||||
<img
|
||||
style={{
|
||||
width: '100%',
|
||||
height: '100%',
|
||||
padding: 7,
|
||||
borderRadius: '50%',
|
||||
objectFit: 'contain'
|
||||
}}
|
||||
alt={loader.name}
|
||||
src={`${baseURL}/api/v1/node-icon/${loader.name}`}
|
||||
/>
|
||||
</div>
|
||||
<Typography>{loader.label}</Typography>
|
||||
</ListItemButton>
|
||||
))}
|
||||
</List>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
) : null
|
||||
|
||||
return createPortal(component, portalElement)
|
||||
}
|
||||
|
||||
ComponentsListDialog.propTypes = {
|
||||
show: PropTypes.bool,
|
||||
dialogProps: PropTypes.object,
|
||||
onCancel: PropTypes.func,
|
||||
apiCall: PropTypes.func,
|
||||
onSelected: PropTypes.func
|
||||
}
|
||||
|
||||
export default ComponentsListDialog
|
||||
@@ -0,0 +1,246 @@
|
||||
import { useState, useEffect } from 'react'
|
||||
import { createPortal } from 'react-dom'
|
||||
import PropTypes from 'prop-types'
|
||||
import { cloneDeep } from 'lodash'
|
||||
import {
|
||||
Button,
|
||||
Box,
|
||||
Paper,
|
||||
Accordion,
|
||||
AccordionSummary,
|
||||
AccordionDetails,
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogTitle,
|
||||
Typography,
|
||||
Table,
|
||||
TableBody,
|
||||
TableContainer,
|
||||
TableRow,
|
||||
TableCell,
|
||||
Checkbox,
|
||||
FormControlLabel,
|
||||
DialogActions
|
||||
} from '@mui/material'
|
||||
import ExpandMoreIcon from '@mui/icons-material/ExpandMore'
|
||||
import { TableViewOnly } from '@/ui-component/table/Table'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
|
||||
// const
|
||||
import { baseURL } from '@/store/constant'
|
||||
import nodesApi from '@/api/nodes'
|
||||
|
||||
// Hooks
|
||||
import useApi from '@/hooks/useApi'
|
||||
import { initNode } from '@/utils/genericHelper'
|
||||
|
||||
const DeleteDocStoreDialog = ({ show, dialogProps, onCancel, onDelete }) => {
|
||||
const portalElement = document.getElementById('portal')
|
||||
const [nodeConfigExpanded, setNodeConfigExpanded] = useState({})
|
||||
const [removeFromVS, setRemoveFromVS] = useState(false)
|
||||
const [vsFlowData, setVSFlowData] = useState([])
|
||||
const [rmFlowData, setRMFlowData] = useState([])
|
||||
|
||||
const getSpecificNodeApi = useApi(nodesApi.getSpecificNode)
|
||||
|
||||
const handleAccordionChange = (nodeName) => (event, isExpanded) => {
|
||||
const accordianNodes = { ...nodeConfigExpanded }
|
||||
accordianNodes[nodeName] = isExpanded
|
||||
setNodeConfigExpanded(accordianNodes)
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
if (dialogProps.recordManagerConfig) {
|
||||
const nodeName = dialogProps.recordManagerConfig.name
|
||||
if (nodeName) getSpecificNodeApi.request(nodeName)
|
||||
|
||||
if (dialogProps.vectorStoreConfig) {
|
||||
const nodeName = dialogProps.vectorStoreConfig.name
|
||||
if (nodeName) getSpecificNodeApi.request(nodeName)
|
||||
}
|
||||
}
|
||||
|
||||
return () => {
|
||||
setNodeConfigExpanded({})
|
||||
setRemoveFromVS(false)
|
||||
setVSFlowData([])
|
||||
setRMFlowData([])
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [dialogProps])
|
||||
|
||||
useEffect(() => {
|
||||
if (getSpecificNodeApi.data) {
|
||||
const nodeData = cloneDeep(initNode(getSpecificNodeApi.data, uuidv4()))
|
||||
|
||||
let config = 'vectorStoreConfig'
|
||||
if (nodeData.category === 'Record Manager') config = 'recordManagerConfig'
|
||||
|
||||
const paramValues = []
|
||||
|
||||
for (const inputName in dialogProps[config].config) {
|
||||
const inputParam = nodeData.inputParams.find((inp) => inp.name === inputName)
|
||||
|
||||
if (!inputParam) continue
|
||||
|
||||
if (inputParam.type === 'credential') continue
|
||||
|
||||
let paramValue = {}
|
||||
|
||||
const inputValue = dialogProps[config].config[inputName]
|
||||
|
||||
if (!inputValue) continue
|
||||
|
||||
if (typeof inputValue === 'string' && inputValue.startsWith('{{') && inputValue.endsWith('}}')) {
|
||||
continue
|
||||
}
|
||||
|
||||
paramValue = {
|
||||
label: inputParam?.label,
|
||||
name: inputParam?.name,
|
||||
type: inputParam?.type,
|
||||
value: inputValue
|
||||
}
|
||||
paramValues.push(paramValue)
|
||||
}
|
||||
|
||||
if (config === 'vectorStoreConfig') {
|
||||
setVSFlowData([
|
||||
{
|
||||
label: nodeData.label,
|
||||
name: nodeData.name,
|
||||
category: nodeData.category,
|
||||
id: nodeData.id,
|
||||
paramValues
|
||||
}
|
||||
])
|
||||
} else if (config === 'recordManagerConfig') {
|
||||
setRMFlowData([
|
||||
{
|
||||
label: nodeData.label,
|
||||
name: nodeData.name,
|
||||
category: nodeData.category,
|
||||
id: nodeData.id,
|
||||
paramValues
|
||||
}
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [getSpecificNodeApi.data])
|
||||
|
||||
const component = show ? (
|
||||
<Dialog
|
||||
fullWidth
|
||||
maxWidth={dialogProps.recordManagerConfig ? 'md' : 'sm'}
|
||||
open={show}
|
||||
onClose={onCancel}
|
||||
aria-labelledby='alert-dialog-title'
|
||||
aria-describedby='alert-dialog-description'
|
||||
>
|
||||
<DialogTitle sx={{ fontSize: '1rem', p: 3, pb: 0 }} id='alert-dialog-title'>
|
||||
{dialogProps.title}
|
||||
</DialogTitle>
|
||||
<DialogContent sx={{ display: 'flex', flexDirection: 'column', gap: 2, maxHeight: '75vh', position: 'relative', px: 3, pb: 3 }}>
|
||||
<span style={{ marginTop: '20px' }}>{dialogProps.description}</span>
|
||||
{dialogProps.type === 'STORE' && dialogProps.recordManagerConfig && (
|
||||
<FormControlLabel
|
||||
control={<Checkbox checked={removeFromVS} onChange={(event) => setRemoveFromVS(event.target.checked)} />}
|
||||
label='Remove data from vector store'
|
||||
/>
|
||||
)}
|
||||
{removeFromVS && (
|
||||
<div>
|
||||
<TableContainer component={Paper}>
|
||||
<Table sx={{ minWidth: 650 }} aria-label='simple table'>
|
||||
<TableBody>
|
||||
<TableRow sx={{ '& td': { border: 0 } }}>
|
||||
<TableCell sx={{ pb: 0, pt: 0 }} colSpan={6}>
|
||||
<Box>
|
||||
{([...vsFlowData, ...rmFlowData] || []).map((node, index) => {
|
||||
return (
|
||||
<Accordion
|
||||
expanded={nodeConfigExpanded[node.name] || true}
|
||||
onChange={handleAccordionChange(node.name)}
|
||||
key={index}
|
||||
disableGutters
|
||||
>
|
||||
<AccordionSummary
|
||||
expandIcon={<ExpandMoreIcon />}
|
||||
aria-controls={`nodes-accordian-${node.name}`}
|
||||
id={`nodes-accordian-header-${node.name}`}
|
||||
>
|
||||
<div
|
||||
style={{ display: 'flex', flexDirection: 'row', alignItems: 'center' }}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
width: 40,
|
||||
height: 40,
|
||||
marginRight: 10,
|
||||
borderRadius: '50%',
|
||||
backgroundColor: 'white'
|
||||
}}
|
||||
>
|
||||
<img
|
||||
style={{
|
||||
width: '100%',
|
||||
height: '100%',
|
||||
padding: 7,
|
||||
borderRadius: '50%',
|
||||
objectFit: 'contain'
|
||||
}}
|
||||
alt={node.name}
|
||||
src={`${baseURL}/api/v1/node-icon/${node.name}`}
|
||||
/>
|
||||
</div>
|
||||
<Typography variant='h5'>{node.label}</Typography>
|
||||
</div>
|
||||
</AccordionSummary>
|
||||
<AccordionDetails>
|
||||
{node.paramValues[0] && (
|
||||
<TableViewOnly
|
||||
sx={{ minWidth: 150 }}
|
||||
rows={node.paramValues}
|
||||
columns={Object.keys(node.paramValues[0])}
|
||||
/>
|
||||
)}
|
||||
</AccordionDetails>
|
||||
</Accordion>
|
||||
)
|
||||
})}
|
||||
</Box>
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
</TableBody>
|
||||
</Table>
|
||||
</TableContainer>
|
||||
<span style={{ marginTop: '30px', fontStyle: 'italic', color: '#b35702' }}>
|
||||
* Only data that were upserted with Record Manager will be deleted from vector store
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
</DialogContent>
|
||||
<DialogActions sx={{ pr: 3, pb: 3 }}>
|
||||
<Button onClick={onCancel} color='primary'>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button variant='contained' onClick={() => onDelete(dialogProps.type, removeFromVS)} color='error'>
|
||||
Delete
|
||||
</Button>
|
||||
</DialogActions>
|
||||
</Dialog>
|
||||
) : null
|
||||
|
||||
return createPortal(component, portalElement)
|
||||
}
|
||||
|
||||
DeleteDocStoreDialog.propTypes = {
|
||||
show: PropTypes.bool,
|
||||
dialogProps: PropTypes.object,
|
||||
onCancel: PropTypes.func,
|
||||
onDelete: PropTypes.func
|
||||
}
|
||||
|
||||
export default DeleteDocStoreDialog
|
||||
@@ -118,8 +118,9 @@ const DocStoreInputHandler = ({ inputParam, data, disabled = false }) => {
|
||||
)}
|
||||
{inputParam.type === 'credential' && (
|
||||
<CredentialInputHandler
|
||||
key={JSON.stringify(inputParam)}
|
||||
disabled={disabled}
|
||||
data={data}
|
||||
data={data.inputs.credential ? { credential: data.inputs.credential } : {}}
|
||||
inputParam={inputParam}
|
||||
onSelect={(newValue) => {
|
||||
data.credential = newValue
|
||||
@@ -189,6 +190,7 @@ const DocStoreInputHandler = ({ inputParam, data, disabled = false }) => {
|
||||
)}
|
||||
{inputParam.type === 'options' && (
|
||||
<Dropdown
|
||||
key={JSON.stringify(inputParam)}
|
||||
disabled={disabled}
|
||||
name={inputParam.name}
|
||||
options={inputParam.options}
|
||||
@@ -198,6 +200,7 @@ const DocStoreInputHandler = ({ inputParam, data, disabled = false }) => {
|
||||
)}
|
||||
{inputParam.type === 'multiOptions' && (
|
||||
<MultiDropdown
|
||||
key={JSON.stringify(inputParam)}
|
||||
disabled={disabled}
|
||||
name={inputParam.name}
|
||||
options={inputParam.options}
|
||||
@@ -207,9 +210,10 @@ const DocStoreInputHandler = ({ inputParam, data, disabled = false }) => {
|
||||
)}
|
||||
{inputParam.type === 'asyncOptions' && (
|
||||
<>
|
||||
{data.inputParams.length === 1 && <div style={{ marginTop: 10 }} />}
|
||||
{data.inputParams?.length === 1 && <div style={{ marginTop: 10 }} />}
|
||||
<div style={{ display: 'flex', flexDirection: 'row' }}>
|
||||
<AsyncDropdown
|
||||
key={JSON.stringify(inputParam)}
|
||||
disabled={disabled}
|
||||
name={inputParam.name}
|
||||
nodeData={data}
|
||||
|
||||
@@ -29,20 +29,31 @@ import { tableCellClasses } from '@mui/material/TableCell'
|
||||
// project imports
|
||||
import MainCard from '@/ui-component/cards/MainCard'
|
||||
import AddDocStoreDialog from '@/views/docstore/AddDocStoreDialog'
|
||||
import ConfirmDialog from '@/ui-component/dialog/ConfirmDialog'
|
||||
import { BackdropLoader } from '@/ui-component/loading/BackdropLoader'
|
||||
import DocumentLoaderListDialog from '@/views/docstore/DocumentLoaderListDialog'
|
||||
import ErrorBoundary from '@/ErrorBoundary'
|
||||
import { StyledButton } from '@/ui-component/button/StyledButton'
|
||||
import ViewHeader from '@/layout/MainLayout/ViewHeader'
|
||||
import DeleteDocStoreDialog from './DeleteDocStoreDialog'
|
||||
|
||||
// API
|
||||
import documentsApi from '@/api/documentstore'
|
||||
|
||||
// Hooks
|
||||
import useApi from '@/hooks/useApi'
|
||||
import useConfirm from '@/hooks/useConfirm'
|
||||
import useNotifier from '@/utils/useNotifier'
|
||||
|
||||
// icons
|
||||
import { IconPlus, IconRefresh, IconScissors, IconTrash, IconX, IconVectorBezier2 } from '@tabler/icons-react'
|
||||
import {
|
||||
IconPlus,
|
||||
IconRefresh,
|
||||
IconListDetails,
|
||||
IconTrash,
|
||||
IconX,
|
||||
IconVectorBezier2,
|
||||
IconRowInsertTop,
|
||||
IconZoomScan
|
||||
} from '@tabler/icons-react'
|
||||
import KeyboardArrowDownIcon from '@mui/icons-material/KeyboardArrowDown'
|
||||
import FileDeleteIcon from '@mui/icons-material/Delete'
|
||||
import FileEditIcon from '@mui/icons-material/Edit'
|
||||
@@ -51,8 +62,6 @@ import doc_store_details_emptySVG from '@/assets/images/doc_store_details_empty.
|
||||
|
||||
// store
|
||||
import { closeSnackbar as closeSnackbarAction, enqueueSnackbar as enqueueSnackbarAction } from '@/store/actions'
|
||||
import { StyledButton } from '@/ui-component/button/StyledButton'
|
||||
import ViewHeader from '@/layout/MainLayout/ViewHeader'
|
||||
|
||||
// ==============================|| DOCUMENTS ||============================== //
|
||||
|
||||
@@ -121,17 +130,19 @@ const DocumentStoreDetails = () => {
|
||||
|
||||
const enqueueSnackbar = (...args) => dispatch(enqueueSnackbarAction(...args))
|
||||
const closeSnackbar = (...args) => dispatch(closeSnackbarAction(...args))
|
||||
const { confirm } = useConfirm()
|
||||
|
||||
const getSpecificDocumentStore = useApi(documentsApi.getSpecificDocumentStore)
|
||||
|
||||
const [error, setError] = useState(null)
|
||||
const [isLoading, setLoading] = useState(true)
|
||||
const [isBackdropLoading, setBackdropLoading] = useState(false)
|
||||
const [showDialog, setShowDialog] = useState(false)
|
||||
const [documentStore, setDocumentStore] = useState({})
|
||||
const [dialogProps, setDialogProps] = useState({})
|
||||
const [showDocumentLoaderListDialog, setShowDocumentLoaderListDialog] = useState(false)
|
||||
const [documentLoaderListDialogProps, setDocumentLoaderListDialogProps] = useState({})
|
||||
const [showDeleteDocStoreDialog, setShowDeleteDocStoreDialog] = useState(false)
|
||||
const [deleteDocStoreDialogProps, setDeleteDocStoreDialogProps] = useState({})
|
||||
|
||||
const URLpath = document.location.pathname.toString().split('/')
|
||||
const storeId = URLpath[URLpath.length - 1] === 'document-stores' ? '' : URLpath[URLpath.length - 1]
|
||||
@@ -144,11 +155,19 @@ const DocumentStoreDetails = () => {
|
||||
navigate('/document-stores/chunks/' + storeId + '/' + id)
|
||||
}
|
||||
|
||||
const showVectorStoreQuery = (id) => {
|
||||
navigate('/document-stores/query/' + id)
|
||||
}
|
||||
|
||||
const onDocLoaderSelected = (docLoaderComponentName) => {
|
||||
setShowDocumentLoaderListDialog(false)
|
||||
navigate('/document-stores/' + storeId + '/' + docLoaderComponentName)
|
||||
}
|
||||
|
||||
const showVectorStore = (id) => {
|
||||
navigate('/document-stores/vector/' + id)
|
||||
}
|
||||
|
||||
const listLoaders = () => {
|
||||
const dialogProp = {
|
||||
title: 'Select Document Loader'
|
||||
@@ -157,18 +176,60 @@ const DocumentStoreDetails = () => {
|
||||
setShowDocumentLoaderListDialog(true)
|
||||
}
|
||||
|
||||
const onLoaderDelete = async (file) => {
|
||||
const confirmPayload = {
|
||||
title: `Delete`,
|
||||
description: `Delete Loader ${file.loaderName} ? This will delete all the associated document chunks.`,
|
||||
confirmButtonName: 'Delete',
|
||||
cancelButtonName: 'Cancel'
|
||||
const deleteVectorStoreDataFromStore = async (storeId) => {
|
||||
try {
|
||||
await documentsApi.deleteVectorStoreDataFromStore(storeId)
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
}
|
||||
const isConfirmed = await confirm(confirmPayload)
|
||||
}
|
||||
|
||||
if (isConfirmed) {
|
||||
const onDocStoreDelete = async (type, removeFromVectorStore) => {
|
||||
setBackdropLoading(true)
|
||||
if (type === 'STORE') {
|
||||
if (removeFromVectorStore) {
|
||||
await deleteVectorStoreDataFromStore(storeId)
|
||||
}
|
||||
try {
|
||||
const deleteResp = await documentsApi.deleteDocumentStore(storeId)
|
||||
setBackdropLoading(false)
|
||||
if (deleteResp.data) {
|
||||
enqueueSnackbar({
|
||||
message: 'Store, Loader and associated document chunks deleted',
|
||||
options: {
|
||||
key: new Date().getTime() + Math.random(),
|
||||
variant: 'success',
|
||||
action: (key) => (
|
||||
<Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
|
||||
<IconX />
|
||||
</Button>
|
||||
)
|
||||
}
|
||||
})
|
||||
navigate('/document-stores/')
|
||||
}
|
||||
} catch (error) {
|
||||
setBackdropLoading(false)
|
||||
setError(error)
|
||||
const errorData = error.response.data || `${error.response.status}: ${error.response.statusText}`
|
||||
enqueueSnackbar({
|
||||
message: `Failed to delete loader: ${errorData}`,
|
||||
options: {
|
||||
key: new Date().getTime() + Math.random(),
|
||||
variant: 'error',
|
||||
persist: true,
|
||||
action: (key) => (
|
||||
<Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
|
||||
<IconX />
|
||||
</Button>
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
} else if (type === 'LOADER') {
|
||||
try {
|
||||
const deleteResp = await documentsApi.deleteLoaderFromStore(storeId, file.id)
|
||||
setBackdropLoading(false)
|
||||
if (deleteResp.data) {
|
||||
enqueueSnackbar({
|
||||
message: 'Loader and associated document chunks deleted',
|
||||
@@ -186,6 +247,7 @@ const DocumentStoreDetails = () => {
|
||||
}
|
||||
} catch (error) {
|
||||
setError(error)
|
||||
setBackdropLoading(false)
|
||||
const errorData = error.response.data || `${error.response.status}: ${error.response.statusText}`
|
||||
enqueueSnackbar({
|
||||
message: `Failed to delete loader: ${errorData}`,
|
||||
@@ -204,51 +266,30 @@ const DocumentStoreDetails = () => {
|
||||
}
|
||||
}
|
||||
|
||||
const onStoreDelete = async () => {
|
||||
const confirmPayload = {
|
||||
const onLoaderDelete = (file, vectorStoreConfig, recordManagerConfig) => {
|
||||
const props = {
|
||||
title: `Delete`,
|
||||
description: `Delete Loader ${file.loaderName} ? This will delete all the associated document chunks.`,
|
||||
vectorStoreConfig,
|
||||
recordManagerConfig,
|
||||
type: 'LOADER'
|
||||
}
|
||||
|
||||
setDeleteDocStoreDialogProps(props)
|
||||
setShowDeleteDocStoreDialog(true)
|
||||
}
|
||||
|
||||
const onStoreDelete = (vectorStoreConfig, recordManagerConfig) => {
|
||||
const props = {
|
||||
title: `Delete`,
|
||||
description: `Delete Store ${getSpecificDocumentStore.data?.name} ? This will delete all the associated loaders and document chunks.`,
|
||||
confirmButtonName: 'Delete',
|
||||
cancelButtonName: 'Cancel'
|
||||
vectorStoreConfig,
|
||||
recordManagerConfig,
|
||||
type: 'STORE'
|
||||
}
|
||||
const isConfirmed = await confirm(confirmPayload)
|
||||
|
||||
if (isConfirmed) {
|
||||
try {
|
||||
const deleteResp = await documentsApi.deleteDocumentStore(storeId)
|
||||
if (deleteResp.data) {
|
||||
enqueueSnackbar({
|
||||
message: 'Store, Loader and associated document chunks deleted',
|
||||
options: {
|
||||
key: new Date().getTime() + Math.random(),
|
||||
variant: 'success',
|
||||
action: (key) => (
|
||||
<Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
|
||||
<IconX />
|
||||
</Button>
|
||||
)
|
||||
}
|
||||
})
|
||||
navigate('/document-stores/')
|
||||
}
|
||||
} catch (error) {
|
||||
setError(error)
|
||||
const errorData = error.response.data || `${error.response.status}: ${error.response.statusText}`
|
||||
enqueueSnackbar({
|
||||
message: `Failed to delete loader: ${errorData}`,
|
||||
options: {
|
||||
key: new Date().getTime() + Math.random(),
|
||||
variant: 'error',
|
||||
persist: true,
|
||||
action: (key) => (
|
||||
<Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
|
||||
<IconX />
|
||||
</Button>
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
setDeleteDocStoreDialogProps(props)
|
||||
setShowDeleteDocStoreDialog(true)
|
||||
}
|
||||
|
||||
const onEditClicked = () => {
|
||||
@@ -315,24 +356,15 @@ const DocumentStoreDetails = () => {
|
||||
onBack={() => navigate('/document-stores')}
|
||||
onEdit={() => onEditClicked()}
|
||||
>
|
||||
<IconButton onClick={onStoreDelete} size='small' color='error' title='Delete Document Store' sx={{ mr: 2 }}>
|
||||
<IconButton
|
||||
onClick={() => onStoreDelete(documentStore.vectorStoreConfig, documentStore.recordManagerConfig)}
|
||||
size='small'
|
||||
color='error'
|
||||
title='Delete Document Store'
|
||||
sx={{ mr: 2 }}
|
||||
>
|
||||
<IconTrash />
|
||||
</IconButton>
|
||||
{documentStore?.status === 'STALE' && (
|
||||
<Button variant='outlined' sx={{ mr: 2 }} startIcon={<IconRefresh />} onClick={onConfirm}>
|
||||
Refresh
|
||||
</Button>
|
||||
)}
|
||||
{documentStore?.totalChunks > 0 && (
|
||||
<Button
|
||||
variant='outlined'
|
||||
sx={{ borderRadius: 2, height: '100%' }}
|
||||
startIcon={<IconScissors />}
|
||||
onClick={() => showStoredChunks('all')}
|
||||
>
|
||||
View Chunks
|
||||
</Button>
|
||||
)}
|
||||
<StyledButton
|
||||
variant='contained'
|
||||
sx={{ borderRadius: 2, height: '100%', color: 'white' }}
|
||||
@@ -341,6 +373,67 @@ const DocumentStoreDetails = () => {
|
||||
>
|
||||
Add Document Loader
|
||||
</StyledButton>
|
||||
{(documentStore?.status === 'STALE' || documentStore?.status === 'UPSERTING') && (
|
||||
<Button variant='outlined' sx={{ mr: 2 }} startIcon={<IconRefresh />} onClick={onConfirm}>
|
||||
Refresh
|
||||
</Button>
|
||||
)}
|
||||
{documentStore?.status === 'UPSERTING' && (
|
||||
<Chip
|
||||
variant='raised'
|
||||
label='Upserting to Vector Store'
|
||||
color='warning'
|
||||
sx={{ borderRadius: 2, height: '100%' }}
|
||||
/>
|
||||
)}
|
||||
{documentStore?.totalChunks > 0 && documentStore?.status !== 'UPSERTING' && (
|
||||
<>
|
||||
<Button
|
||||
variant='contained'
|
||||
sx={{
|
||||
borderRadius: 2,
|
||||
height: '100%'
|
||||
}}
|
||||
color='secondary'
|
||||
startIcon={<IconListDetails />}
|
||||
onClick={() => showStoredChunks('all')}
|
||||
>
|
||||
View Chunks
|
||||
</Button>
|
||||
<Button
|
||||
variant='contained'
|
||||
sx={{
|
||||
borderRadius: 2,
|
||||
height: '100%',
|
||||
backgroundImage: `linear-gradient(to right, #13547a, #2f9e91)`,
|
||||
'&:hover': {
|
||||
backgroundImage: `linear-gradient(to right, #0b3d5b, #1a8377)`
|
||||
}
|
||||
}}
|
||||
startIcon={<IconRowInsertTop />}
|
||||
onClick={() => showVectorStore(documentStore.id)}
|
||||
>
|
||||
Upsert Config
|
||||
</Button>
|
||||
</>
|
||||
)}
|
||||
{documentStore?.totalChunks > 0 && documentStore?.status === 'UPSERTED' && (
|
||||
<Button
|
||||
variant='contained'
|
||||
sx={{
|
||||
borderRadius: 2,
|
||||
height: '100%',
|
||||
backgroundImage: `linear-gradient(to right, #3f5efb, #fc466b)`,
|
||||
'&:hover': {
|
||||
backgroundImage: `linear-gradient(to right, #2b4efb, #fe2752)`
|
||||
}
|
||||
}}
|
||||
startIcon={<IconZoomScan />}
|
||||
onClick={() => showVectorStoreQuery(documentStore.id)}
|
||||
>
|
||||
Retrieval Query
|
||||
</Button>
|
||||
)}
|
||||
</ViewHeader>
|
||||
{getSpecificDocumentStore.data?.whereUsed?.length > 0 && (
|
||||
<Stack flexDirection='row' sx={{ gap: 2, alignItems: 'center', flexWrap: 'wrap' }}>
|
||||
@@ -482,7 +575,13 @@ const DocumentStoreDetails = () => {
|
||||
theme={theme}
|
||||
onEditClick={() => openPreviewSettings(loader.id)}
|
||||
onViewChunksClick={() => showStoredChunks(loader.id)}
|
||||
onDeleteClick={() => onLoaderDelete(loader)}
|
||||
onDeleteClick={() =>
|
||||
onLoaderDelete(
|
||||
loader,
|
||||
documentStore?.vectorStoreConfig,
|
||||
documentStore?.recordManagerConfig
|
||||
)
|
||||
}
|
||||
/>
|
||||
))}
|
||||
</>
|
||||
@@ -520,7 +619,15 @@ const DocumentStoreDetails = () => {
|
||||
onDocLoaderSelected={onDocLoaderSelected}
|
||||
/>
|
||||
)}
|
||||
<ConfirmDialog />
|
||||
{showDeleteDocStoreDialog && (
|
||||
<DeleteDocStoreDialog
|
||||
show={showDeleteDocStoreDialog}
|
||||
dialogProps={deleteDocStoreDialogProps}
|
||||
onCancel={() => setShowDeleteDocStoreDialog(false)}
|
||||
onDelete={onDocStoreDelete}
|
||||
/>
|
||||
)}
|
||||
{isBackdropLoading && <BackdropLoader open={isBackdropLoading} />}
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -15,8 +15,10 @@ const DocumentStoreStatus = ({ status, isTableView }) => {
|
||||
case 'EMPTY':
|
||||
return ['#673ab7', '#673ab7', '#673ab7']
|
||||
case 'SYNCING':
|
||||
case 'UPSERTING':
|
||||
return ['#fff8e1', '#ffe57f', '#ffc107']
|
||||
case 'SYNC':
|
||||
case 'UPSERTED':
|
||||
return ['#cdf5d8', '#00e676', '#00c853']
|
||||
case 'NEW':
|
||||
return ['#e3f2fd', '#2196f3', '#1e88e5']
|
||||
|
||||
@@ -279,7 +279,6 @@ const LoaderConfigPreviewChunks = () => {
|
||||
useEffect(() => {
|
||||
if (getNodeDetailsApi.data) {
|
||||
const nodeData = cloneDeep(initNode(getNodeDetailsApi.data, uuidv4()))
|
||||
|
||||
// If this is a document store edit config, set the existing input values
|
||||
if (existingLoaderFromDocStoreTable && existingLoaderFromDocStoreTable.loaderConfig) {
|
||||
nodeData.inputs = existingLoaderFromDocStoreTable.loaderConfig
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
import { useState } from 'react'
|
||||
import { createPortal } from 'react-dom'
|
||||
import PropTypes from 'prop-types'
|
||||
import {
|
||||
Box,
|
||||
Paper,
|
||||
Accordion,
|
||||
AccordionSummary,
|
||||
AccordionDetails,
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogTitle,
|
||||
Typography,
|
||||
Table,
|
||||
TableBody,
|
||||
TableContainer,
|
||||
TableRow,
|
||||
TableCell
|
||||
} from '@mui/material'
|
||||
import ExpandMoreIcon from '@mui/icons-material/ExpandMore'
|
||||
import { TableViewOnly } from '@/ui-component/table/Table'
|
||||
import StatsCard from '@/ui-component/cards/StatsCard'
|
||||
|
||||
// const
|
||||
import { baseURL } from '@/store/constant'
|
||||
|
||||
const UpsertHistoryDetailsDialog = ({ show, dialogProps, onCancel }) => {
|
||||
const portalElement = document.getElementById('portal')
|
||||
const [nodeConfigExpanded, setNodeConfigExpanded] = useState({})
|
||||
|
||||
const handleAccordionChange = (nodeLabel) => (event, isExpanded) => {
|
||||
const accordianNodes = { ...nodeConfigExpanded }
|
||||
accordianNodes[nodeLabel] = isExpanded
|
||||
setNodeConfigExpanded(accordianNodes)
|
||||
}
|
||||
const component = show ? (
|
||||
<Dialog
|
||||
fullWidth
|
||||
maxWidth='md'
|
||||
open={show}
|
||||
onClose={onCancel}
|
||||
aria-labelledby='alert-dialog-title'
|
||||
aria-describedby='alert-dialog-description'
|
||||
>
|
||||
<DialogTitle sx={{ fontSize: '1rem', p: 3, pb: 0 }} id='alert-dialog-title'>
|
||||
{dialogProps.title}
|
||||
</DialogTitle>
|
||||
<DialogContent sx={{ display: 'flex', flexDirection: 'column', gap: 2, maxHeight: '75vh', position: 'relative', px: 3, pb: 3 }}>
|
||||
<div
|
||||
style={{
|
||||
display: 'grid',
|
||||
gridTemplateColumns: 'repeat(4, minmax(0, 1fr))',
|
||||
gap: 5,
|
||||
marginTop: '10px'
|
||||
}}
|
||||
>
|
||||
<StatsCard title='Added' stat={dialogProps.numAdded ?? 0} />
|
||||
<StatsCard title='Updated' stat={dialogProps.numUpdated ?? 0} />
|
||||
<StatsCard title='Skipped' stat={dialogProps.numSkipped ?? 0} />
|
||||
<StatsCard title='Deleted' stat={dialogProps.numDeleted ?? 0} />
|
||||
</div>
|
||||
<div>
|
||||
<TableContainer component={Paper}>
|
||||
<Table sx={{ minWidth: 650 }} aria-label='simple table'>
|
||||
<TableBody>
|
||||
<TableRow sx={{ '& td': { border: 0 } }}>
|
||||
<TableCell sx={{ pb: 0, pt: 0 }} colSpan={6}>
|
||||
<Box sx={{ mt: 1, mb: 2 }}>
|
||||
{(dialogProps.flowData ?? []).map((node, index) => {
|
||||
return (
|
||||
<Accordion
|
||||
expanded={nodeConfigExpanded[node.id] || false}
|
||||
onChange={handleAccordionChange(node.id)}
|
||||
key={index}
|
||||
disableGutters
|
||||
>
|
||||
<AccordionSummary
|
||||
expandIcon={<ExpandMoreIcon />}
|
||||
aria-controls={`nodes-accordian-${node.name}`}
|
||||
id={`nodes-accordian-header-${node.name}`}
|
||||
>
|
||||
<div style={{ display: 'flex', flexDirection: 'row', alignItems: 'center' }}>
|
||||
<div
|
||||
style={{
|
||||
width: 40,
|
||||
height: 40,
|
||||
marginRight: 10,
|
||||
borderRadius: '50%',
|
||||
backgroundColor: 'white'
|
||||
}}
|
||||
>
|
||||
<img
|
||||
style={{
|
||||
width: '100%',
|
||||
height: '100%',
|
||||
padding: 7,
|
||||
borderRadius: '50%',
|
||||
objectFit: 'contain'
|
||||
}}
|
||||
alt={node.name}
|
||||
src={`${baseURL}/api/v1/node-icon/${node.name}`}
|
||||
/>
|
||||
</div>
|
||||
<Typography variant='h5'>{node.label}</Typography>
|
||||
<div
|
||||
style={{
|
||||
display: 'flex',
|
||||
flexDirection: 'row',
|
||||
width: 'max-content',
|
||||
borderRadius: 15,
|
||||
background: 'rgb(254,252,191)',
|
||||
padding: 5,
|
||||
paddingLeft: 10,
|
||||
paddingRight: 10,
|
||||
marginLeft: 10
|
||||
}}
|
||||
>
|
||||
<span style={{ color: 'rgb(116,66,16)', fontSize: '0.825rem' }}>
|
||||
{node.id}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</AccordionSummary>
|
||||
<AccordionDetails>
|
||||
{node.paramValues[0] && (
|
||||
<TableViewOnly
|
||||
sx={{ minWidth: 150 }}
|
||||
rows={node.paramValues}
|
||||
columns={Object.keys(node.paramValues[0])}
|
||||
/>
|
||||
)}
|
||||
</AccordionDetails>
|
||||
</Accordion>
|
||||
)
|
||||
})}
|
||||
</Box>
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
</TableBody>
|
||||
</Table>
|
||||
</TableContainer>
|
||||
</div>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
) : null
|
||||
|
||||
return createPortal(component, portalElement)
|
||||
}
|
||||
|
||||
UpsertHistoryDetailsDialog.propTypes = {
|
||||
show: PropTypes.bool,
|
||||
dialogProps: PropTypes.object,
|
||||
onCancel: PropTypes.func
|
||||
}
|
||||
|
||||
export default UpsertHistoryDetailsDialog
|
||||
@@ -0,0 +1,113 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
import PropTypes from 'prop-types'
|
||||
import moment from 'moment/moment'
|
||||
|
||||
import { Stack, Button, Box, SwipeableDrawer } from '@mui/material'
|
||||
import { IconSquareRoundedChevronsRight } from '@tabler/icons-react'
|
||||
import {
|
||||
Timeline,
|
||||
TimelineConnector,
|
||||
TimelineContent,
|
||||
TimelineDot,
|
||||
TimelineItem,
|
||||
TimelineOppositeContent,
|
||||
timelineOppositeContentClasses,
|
||||
TimelineSeparator
|
||||
} from '@mui/lab'
|
||||
import HistoryEmptySVG from '@/assets/images/upsert_history_empty.svg'
|
||||
import vectorstoreApi from '@/api/vectorstore'
|
||||
import useApi from '@/hooks/useApi'
|
||||
|
||||
const UpsertHistorySideDrawer = ({ show, dialogProps, onClickFunction, onSelectHistoryDetails }) => {
|
||||
const onOpen = () => {}
|
||||
const [upsertHistory, setUpsertHistory] = useState([])
|
||||
|
||||
const getUpsertHistoryApi = useApi(vectorstoreApi.getUpsertHistory)
|
||||
|
||||
useEffect(() => {
|
||||
getUpsertHistoryApi.request(dialogProps.id)
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [dialogProps])
|
||||
|
||||
useEffect(() => {
|
||||
if (getUpsertHistoryApi.data) {
|
||||
setUpsertHistory(getUpsertHistoryApi.data)
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [getUpsertHistoryApi.data])
|
||||
|
||||
return (
|
||||
<>
|
||||
<SwipeableDrawer anchor='right' open={show} onClose={() => onClickFunction()} onOpen={onOpen}>
|
||||
<Button startIcon={<IconSquareRoundedChevronsRight />} onClick={() => onClickFunction()}>
|
||||
Close
|
||||
</Button>
|
||||
<Box style={{ width: 350, margin: 10 }} role='presentation' onClick={onClickFunction}>
|
||||
<Timeline
|
||||
sx={{
|
||||
[`& .${timelineOppositeContentClasses.root}`]: {
|
||||
flex: 1
|
||||
}
|
||||
}}
|
||||
>
|
||||
{upsertHistory &&
|
||||
upsertHistory.map((history, index) => (
|
||||
<TimelineItem key={index}>
|
||||
<TimelineOppositeContent>
|
||||
{moment(history.date).format('DD-MMM-YYYY, hh:mm:ss A')}
|
||||
</TimelineOppositeContent>
|
||||
<TimelineSeparator style={{ marginTop: 5 }}>
|
||||
<TimelineDot />
|
||||
{index !== upsertHistory.length - 1 && <TimelineConnector />}
|
||||
</TimelineSeparator>
|
||||
<TimelineContent>
|
||||
{history.result.numAdded !== undefined && history.result.numAdded > 0 && (
|
||||
<Box sx={{ fontWeight: 500 }}>Added: {history.result.numAdded}</Box>
|
||||
)}
|
||||
{history.result.numUpdated !== undefined && history.result.numUpdated > 0 && (
|
||||
<Box sx={{ fontWeight: 500 }}>Updated: {history.result.numUpdated}</Box>
|
||||
)}
|
||||
{history.result.numSkipped !== undefined && history.result.numSkipped > 0 && (
|
||||
<Box sx={{ fontWeight: 500 }}>Skipped: {history.result.numSkipped}</Box>
|
||||
)}
|
||||
{history.result.numDeleted !== undefined && history.result.numDeleted > 0 && (
|
||||
<Box sx={{ fontWeight: 500 }}>Deleted: {history.result.numDeleted}</Box>
|
||||
)}
|
||||
<Button
|
||||
size='small'
|
||||
sx={{ mt: 1, mb: 3, borderRadius: '25px' }}
|
||||
variant='outlined'
|
||||
onClick={() => onSelectHistoryDetails(history)}
|
||||
>
|
||||
Details
|
||||
</Button>
|
||||
</TimelineContent>
|
||||
</TimelineItem>
|
||||
))}
|
||||
{upsertHistory.length === 0 && (
|
||||
<Stack sx={{ alignItems: 'center', justifyContent: 'center' }} flexDirection='column'>
|
||||
<Box sx={{ pb: 2, height: 'auto' }}>
|
||||
<img
|
||||
style={{ objectFit: 'cover', height: '10vh', width: 'auto' }}
|
||||
src={HistoryEmptySVG}
|
||||
alt='HistoryEmptySVG'
|
||||
/>
|
||||
</Box>
|
||||
<div>No Upsert History Yet</div>
|
||||
</Stack>
|
||||
)}
|
||||
</Timeline>
|
||||
</Box>
|
||||
</SwipeableDrawer>
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
UpsertHistorySideDrawer.propTypes = {
|
||||
show: PropTypes.bool,
|
||||
dialogProps: PropTypes.object,
|
||||
onClickFunction: PropTypes.func,
|
||||
onSelectHistoryDetails: PropTypes.func
|
||||
}
|
||||
|
||||
export default UpsertHistorySideDrawer
|
||||
@@ -0,0 +1,910 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
import { useDispatch, useSelector } from 'react-redux'
|
||||
import { useNavigate } from 'react-router-dom'
|
||||
import { cloneDeep } from 'lodash'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
import moment from 'moment/moment'
|
||||
|
||||
// material-ui
|
||||
import { Button, Stack, Grid, Box, Typography, IconButton, Stepper, Step, StepLabel } from '@mui/material'
|
||||
|
||||
// project imports
|
||||
import MainCard from '@/ui-component/cards/MainCard'
|
||||
import ConfirmDialog from '@/ui-component/dialog/ConfirmDialog'
|
||||
import ComponentsListDialog from '@/views/docstore/ComponentsListDialog'
|
||||
import DocStoreInputHandler from '@/views/docstore/DocStoreInputHandler'
|
||||
import ViewHeader from '@/layout/MainLayout/ViewHeader'
|
||||
import { BackdropLoader } from '@/ui-component/loading/BackdropLoader'
|
||||
import ErrorBoundary from '@/ErrorBoundary'
|
||||
import UpsertResultDialog from '@/views/vectorstore/UpsertResultDialog'
|
||||
import UpsertHistorySideDrawer from './UpsertHistorySideDrawer'
|
||||
import UpsertHistoryDetailsDialog from './UpsertHistoryDetailsDialog'
|
||||
|
||||
// API
|
||||
import documentsApi from '@/api/documentstore'
|
||||
import nodesApi from '@/api/nodes'
|
||||
|
||||
// Hooks
|
||||
import useApi from '@/hooks/useApi'
|
||||
|
||||
// Store
|
||||
import { closeSnackbar as closeSnackbarAction, enqueueSnackbar as enqueueSnackbarAction } from '@/store/actions'
|
||||
import { baseURL } from '@/store/constant'
|
||||
|
||||
// icons
|
||||
import { IconX, IconEditCircle, IconRowInsertTop, IconDeviceFloppy, IconRefresh, IconClock } from '@tabler/icons-react'
|
||||
import Embeddings from '@mui/icons-material/DynamicFeed'
|
||||
import Storage from '@mui/icons-material/Storage'
|
||||
import DynamicFeed from '@mui/icons-material/Filter1'
|
||||
|
||||
// utils
|
||||
import { initNode } from '@/utils/genericHelper'
|
||||
import useNotifier from '@/utils/useNotifier'
|
||||
|
||||
// const
|
||||
const steps = ['Embeddings', 'Vector Store', 'Record Manager']
|
||||
|
||||
const VectorStoreConfigure = () => {
|
||||
const navigate = useNavigate()
|
||||
const dispatch = useDispatch()
|
||||
useNotifier()
|
||||
const customization = useSelector((state) => state.customization)
|
||||
|
||||
const enqueueSnackbar = (...args) => dispatch(enqueueSnackbarAction(...args))
|
||||
const closeSnackbar = (...args) => dispatch(closeSnackbarAction(...args))
|
||||
|
||||
const getSpecificDocumentStoreApi = useApi(documentsApi.getSpecificDocumentStore)
|
||||
const insertIntoVectorStoreApi = useApi(documentsApi.insertIntoVectorStore)
|
||||
const saveVectorStoreConfigApi = useApi(documentsApi.saveVectorStoreConfig)
|
||||
const getEmbeddingNodeDetailsApi = useApi(nodesApi.getSpecificNode)
|
||||
const getVectorStoreNodeDetailsApi = useApi(nodesApi.getSpecificNode)
|
||||
const getRecordManagerNodeDetailsApi = useApi(nodesApi.getSpecificNode)
|
||||
|
||||
const [error, setError] = useState(null)
|
||||
const [loading, setLoading] = useState(true)
|
||||
|
||||
const [documentStore, setDocumentStore] = useState({})
|
||||
const [dialogProps, setDialogProps] = useState({})
|
||||
|
||||
const [showEmbeddingsListDialog, setShowEmbeddingsListDialog] = useState(false)
|
||||
const [selectedEmbeddingsProvider, setSelectedEmbeddingsProvider] = useState({})
|
||||
|
||||
const [showVectorStoreListDialog, setShowVectorStoreListDialog] = useState(false)
|
||||
const [selectedVectorStoreProvider, setSelectedVectorStoreProvider] = useState({})
|
||||
|
||||
const [showRecordManagerListDialog, setShowRecordManagerListDialog] = useState(false)
|
||||
const [selectedRecordManagerProvider, setSelectedRecordManagerProvider] = useState({})
|
||||
const [isRecordManagerUnavailable, setRecordManagerUnavailable] = useState(false)
|
||||
|
||||
const [showUpsertHistoryDialog, setShowUpsertHistoryDialog] = useState(false)
|
||||
const [upsertResultDialogProps, setUpsertResultDialogProps] = useState({})
|
||||
|
||||
const [showUpsertHistorySideDrawer, setShowUpsertHistorySideDrawer] = useState(false)
|
||||
const [upsertHistoryDrawerDialogProps, setUpsertHistoryDrawerDialogProps] = useState({})
|
||||
|
||||
const [showUpsertHistoryDetailsDialog, setShowUpsertHistoryDetailsDialog] = useState(false)
|
||||
const [upsertDetailsDialogProps, setUpsertDetailsDialogProps] = useState({})
|
||||
|
||||
const onEmbeddingsSelected = (component) => {
|
||||
const nodeData = cloneDeep(initNode(component, uuidv4()))
|
||||
if (!showEmbeddingsListDialog && documentStore.embeddingConfig) {
|
||||
nodeData.inputs = documentStore.embeddingConfig.config
|
||||
nodeData.credential = documentStore.embeddingConfig.config.credential
|
||||
}
|
||||
setSelectedEmbeddingsProvider(nodeData)
|
||||
setShowEmbeddingsListDialog(false)
|
||||
}
|
||||
|
||||
const showEmbeddingsList = () => {
|
||||
const dialogProp = {
|
||||
title: 'Select Embeddings Provider'
|
||||
}
|
||||
setDialogProps(dialogProp)
|
||||
setShowEmbeddingsListDialog(true)
|
||||
}
|
||||
|
||||
const onVectorStoreSelected = (component) => {
|
||||
const nodeData = cloneDeep(initNode(component, uuidv4()))
|
||||
if (!nodeData.inputAnchors.find((anchor) => anchor.name === 'recordManager')) {
|
||||
setRecordManagerUnavailable(true)
|
||||
setSelectedRecordManagerProvider({})
|
||||
} else {
|
||||
setRecordManagerUnavailable(false)
|
||||
}
|
||||
if (!showVectorStoreListDialog && documentStore.vectorStoreConfig) {
|
||||
nodeData.inputs = documentStore.vectorStoreConfig.config
|
||||
nodeData.credential = documentStore.vectorStoreConfig.config.credential
|
||||
}
|
||||
setSelectedVectorStoreProvider(nodeData)
|
||||
setShowVectorStoreListDialog(false)
|
||||
}
|
||||
|
||||
const showVectorStoreList = () => {
|
||||
const dialogProp = {
|
||||
title: 'Select a Vector Store Provider'
|
||||
}
|
||||
setDialogProps(dialogProp)
|
||||
setShowVectorStoreListDialog(true)
|
||||
}
|
||||
|
||||
const onRecordManagerSelected = (component) => {
|
||||
const nodeData = cloneDeep(initNode(component, uuidv4()))
|
||||
if (!showRecordManagerListDialog && documentStore.recordManagerConfig) {
|
||||
nodeData.inputs = documentStore.recordManagerConfig.config
|
||||
nodeData.credential = documentStore.recordManagerConfig.config.credential
|
||||
}
|
||||
setSelectedRecordManagerProvider(nodeData)
|
||||
setShowRecordManagerListDialog(false)
|
||||
}
|
||||
|
||||
const showRecordManagerList = () => {
|
||||
const dialogProp = {
|
||||
title: 'Select a Record Manager'
|
||||
}
|
||||
setDialogProps(dialogProp)
|
||||
setShowRecordManagerListDialog(true)
|
||||
}
|
||||
|
||||
const showUpsertHistoryDrawer = () => {
|
||||
const dialogProp = {
|
||||
id: storeId
|
||||
}
|
||||
setUpsertHistoryDrawerDialogProps(dialogProp)
|
||||
setShowUpsertHistorySideDrawer(true)
|
||||
}
|
||||
|
||||
const onSelectHistoryDetails = (history) => {
|
||||
const props = {
|
||||
title: moment(history.date).format('DD-MMM-YYYY, hh:mm:ss A'),
|
||||
numAdded: history.result.numAdded,
|
||||
numUpdated: history.result.numUpdated,
|
||||
numSkipped: history.result.numSkipped,
|
||||
numDeleted: history.result.numDeleted,
|
||||
flowData: history.flowData
|
||||
}
|
||||
setUpsertDetailsDialogProps(props)
|
||||
setShowUpsertHistoryDetailsDialog(true)
|
||||
}
|
||||
|
||||
const checkMandatoryFields = () => {
|
||||
let canSubmit = true
|
||||
const inputParams = (selectedVectorStoreProvider?.inputParams ?? []).filter((inputParam) => !inputParam.hidden)
|
||||
for (const inputParam of inputParams) {
|
||||
if (!inputParam.optional && (!selectedVectorStoreProvider.inputs[inputParam.name] || !selectedVectorStoreProvider.credential)) {
|
||||
if (inputParam.type === 'credential' && !selectedVectorStoreProvider.credential) {
|
||||
canSubmit = false
|
||||
break
|
||||
} else if (inputParam.type !== 'credential' && !selectedVectorStoreProvider.inputs[inputParam.name]) {
|
||||
canSubmit = false
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const inputParams2 = (selectedEmbeddingsProvider?.inputParams ?? []).filter((inputParam) => !inputParam.hidden)
|
||||
for (const inputParam of inputParams2) {
|
||||
if (!inputParam.optional && (!selectedEmbeddingsProvider.inputs[inputParam.name] || !selectedEmbeddingsProvider.credential)) {
|
||||
if (inputParam.type === 'credential' && !selectedEmbeddingsProvider.credential) {
|
||||
canSubmit = false
|
||||
break
|
||||
} else if (inputParam.type !== 'credential' && !selectedEmbeddingsProvider.inputs[inputParam.name]) {
|
||||
canSubmit = false
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!canSubmit) {
|
||||
enqueueSnackbar({
|
||||
message: 'Please fill in all mandatory fields.',
|
||||
options: {
|
||||
key: new Date().getTime() + Math.random(),
|
||||
variant: 'warning',
|
||||
action: (key) => (
|
||||
<Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
|
||||
<IconX />
|
||||
</Button>
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
return canSubmit
|
||||
}
|
||||
|
||||
const prepareConfigData = () => {
|
||||
const data = {
|
||||
storeId: storeId
|
||||
}
|
||||
// Set embedding config
|
||||
if (selectedEmbeddingsProvider.inputs) {
|
||||
data.embeddingConfig = {}
|
||||
data.embeddingName = selectedEmbeddingsProvider.name
|
||||
Object.keys(selectedEmbeddingsProvider.inputs).map((key) => {
|
||||
if (key === 'FLOWISE_CREDENTIAL_ID') {
|
||||
data.embeddingConfig['credential'] = selectedEmbeddingsProvider.inputs[key]
|
||||
} else {
|
||||
data.embeddingConfig[key] = selectedEmbeddingsProvider.inputs[key]
|
||||
}
|
||||
})
|
||||
} else {
|
||||
data.embeddingConfig = null
|
||||
data.embeddingName = ''
|
||||
}
|
||||
|
||||
// Set vector store config
|
||||
if (selectedVectorStoreProvider.inputs) {
|
||||
data.vectorStoreConfig = {}
|
||||
data.vectorStoreName = selectedVectorStoreProvider.name
|
||||
Object.keys(selectedVectorStoreProvider.inputs).map((key) => {
|
||||
if (key === 'FLOWISE_CREDENTIAL_ID') {
|
||||
data.vectorStoreConfig['credential'] = selectedVectorStoreProvider.inputs[key]
|
||||
} else {
|
||||
data.vectorStoreConfig[key] = selectedVectorStoreProvider.inputs[key]
|
||||
}
|
||||
})
|
||||
} else {
|
||||
data.vectorStoreConfig = null
|
||||
data.vectorStoreName = ''
|
||||
}
|
||||
|
||||
// Set record manager config
|
||||
if (selectedRecordManagerProvider.inputs) {
|
||||
data.recordManagerConfig = {}
|
||||
data.recordManagerName = selectedRecordManagerProvider.name
|
||||
Object.keys(selectedRecordManagerProvider.inputs).map((key) => {
|
||||
if (key === 'FLOWISE_CREDENTIAL_ID') {
|
||||
data.recordManagerConfig['credential'] = selectedRecordManagerProvider.inputs[key]
|
||||
} else {
|
||||
data.recordManagerConfig[key] = selectedRecordManagerProvider.inputs[key]
|
||||
}
|
||||
})
|
||||
} else {
|
||||
data.recordManagerConfig = null
|
||||
data.recordManagerName = ''
|
||||
}
|
||||
|
||||
return data
|
||||
}
|
||||
|
||||
const tryAndInsertIntoStore = () => {
|
||||
if (checkMandatoryFields()) {
|
||||
setLoading(true)
|
||||
const data = prepareConfigData()
|
||||
insertIntoVectorStoreApi.request(data)
|
||||
}
|
||||
}
|
||||
|
||||
const saveVectorStoreConfig = () => {
|
||||
setLoading(true)
|
||||
const data = prepareConfigData()
|
||||
saveVectorStoreConfigApi.request(data)
|
||||
}
|
||||
|
||||
const resetVectorStoreConfig = () => {
|
||||
setSelectedEmbeddingsProvider({})
|
||||
setSelectedVectorStoreProvider({})
|
||||
setSelectedRecordManagerProvider({})
|
||||
}
|
||||
|
||||
const getActiveStep = () => {
|
||||
if (selectedRecordManagerProvider && Object.keys(selectedRecordManagerProvider).length > 0) {
|
||||
return 3
|
||||
}
|
||||
if (selectedVectorStoreProvider && Object.keys(selectedVectorStoreProvider).length > 0) {
|
||||
return 2
|
||||
}
|
||||
if (selectedEmbeddingsProvider && Object.keys(selectedEmbeddingsProvider).length > 0) {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
const Steps = () => {
|
||||
return (
|
||||
<Box sx={{ width: '100%' }}>
|
||||
<Stepper activeStep={getActiveStep()} alternativeLabel>
|
||||
{steps.map((label) => (
|
||||
<Step key={label}>
|
||||
<StepLabel>{label}</StepLabel>
|
||||
</Step>
|
||||
))}
|
||||
</Stepper>
|
||||
</Box>
|
||||
)
|
||||
}
|
||||
|
||||
const isRecordManagerDisabled = () => {
|
||||
return Object.keys(selectedVectorStoreProvider).length === 0 || isRecordManagerUnavailable
|
||||
}
|
||||
|
||||
const isVectorStoreDisabled = () => {
|
||||
return Object.keys(selectedEmbeddingsProvider).length === 0
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
if (saveVectorStoreConfigApi.data) {
|
||||
setLoading(false)
|
||||
enqueueSnackbar({
|
||||
message: 'Configuration saved successfully',
|
||||
options: {
|
||||
key: new Date().getTime() + Math.random(),
|
||||
variant: 'success',
|
||||
action: (key) => (
|
||||
<Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
|
||||
<IconX />
|
||||
</Button>
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [saveVectorStoreConfigApi.data])
|
||||
|
||||
useEffect(() => {
|
||||
if (insertIntoVectorStoreApi.data) {
|
||||
setLoading(false)
|
||||
setShowUpsertHistoryDialog(true)
|
||||
setUpsertResultDialogProps({ ...insertIntoVectorStoreApi.data, goToRetrievalQuery: true })
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [insertIntoVectorStoreApi.data])
|
||||
|
||||
useEffect(() => {
|
||||
if (insertIntoVectorStoreApi.error) {
|
||||
setLoading(false)
|
||||
setError(insertIntoVectorStoreApi.error)
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [insertIntoVectorStoreApi.error])
|
||||
|
||||
useEffect(() => {
|
||||
if (saveVectorStoreConfigApi.error) {
|
||||
setLoading(false)
|
||||
setError(saveVectorStoreConfigApi.error)
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [saveVectorStoreConfigApi.error])
|
||||
|
||||
const URLpath = document.location.pathname.toString().split('/')
|
||||
const storeId = URLpath[URLpath.length - 1] === 'document-stores' ? '' : URLpath[URLpath.length - 1]
|
||||
useEffect(() => {
|
||||
getSpecificDocumentStoreApi.request(storeId)
|
||||
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
if (getSpecificDocumentStoreApi.data) {
|
||||
const docStore = getSpecificDocumentStoreApi.data
|
||||
setDocumentStore(docStore)
|
||||
if (docStore.embeddingConfig) {
|
||||
getEmbeddingNodeDetailsApi.request(docStore.embeddingConfig.name)
|
||||
}
|
||||
if (docStore.vectorStoreConfig) {
|
||||
getVectorStoreNodeDetailsApi.request(docStore.vectorStoreConfig.name)
|
||||
}
|
||||
if (docStore.recordManagerConfig) {
|
||||
getRecordManagerNodeDetailsApi.request(docStore.recordManagerConfig.name)
|
||||
}
|
||||
setLoading(false)
|
||||
}
|
||||
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [getSpecificDocumentStoreApi.data])
|
||||
|
||||
useEffect(() => {
|
||||
if (getEmbeddingNodeDetailsApi.data) {
|
||||
const node = getEmbeddingNodeDetailsApi.data
|
||||
onEmbeddingsSelected(node)
|
||||
}
|
||||
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [getEmbeddingNodeDetailsApi.data])
|
||||
|
||||
useEffect(() => {
|
||||
if (getVectorStoreNodeDetailsApi.data) {
|
||||
const node = getVectorStoreNodeDetailsApi.data
|
||||
onVectorStoreSelected(node)
|
||||
}
|
||||
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [getVectorStoreNodeDetailsApi.data])
|
||||
|
||||
useEffect(() => {
|
||||
if (getRecordManagerNodeDetailsApi.data) {
|
||||
const node = getRecordManagerNodeDetailsApi.data
|
||||
onRecordManagerSelected(node)
|
||||
}
|
||||
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [getRecordManagerNodeDetailsApi.data])
|
||||
|
||||
useEffect(() => {
|
||||
if (getSpecificDocumentStoreApi.error) {
|
||||
setError(getSpecificDocumentStoreApi.error)
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [getSpecificDocumentStoreApi.error])
|
||||
|
||||
return (
|
||||
<>
|
||||
<MainCard>
|
||||
{error ? (
|
||||
<ErrorBoundary error={error} />
|
||||
) : (
|
||||
<Stack flexDirection='column' sx={{ gap: 3 }}>
|
||||
<ViewHeader
|
||||
isBackButton={true}
|
||||
search={false}
|
||||
title={getSpecificDocumentStoreApi.data?.name}
|
||||
description='Configure Embeddings, Vector Store and Record Manager'
|
||||
onBack={() => navigate(-1)}
|
||||
>
|
||||
{(Object.keys(selectedEmbeddingsProvider).length > 0 ||
|
||||
Object.keys(selectedVectorStoreProvider).length > 0) && (
|
||||
<Button
|
||||
variant='outlined'
|
||||
color='error'
|
||||
sx={{
|
||||
borderRadius: 2,
|
||||
height: '100%'
|
||||
}}
|
||||
startIcon={<IconRefresh />}
|
||||
onClick={() => resetVectorStoreConfig()}
|
||||
>
|
||||
Reset
|
||||
</Button>
|
||||
)}
|
||||
{(Object.keys(selectedEmbeddingsProvider).length > 0 ||
|
||||
Object.keys(selectedVectorStoreProvider).length > 0) && (
|
||||
<Button
|
||||
variant='outlined'
|
||||
color='secondary'
|
||||
sx={{
|
||||
borderRadius: 2,
|
||||
height: '100%'
|
||||
}}
|
||||
startIcon={<IconDeviceFloppy />}
|
||||
onClick={() => saveVectorStoreConfig()}
|
||||
>
|
||||
Save Config
|
||||
</Button>
|
||||
)}
|
||||
{Object.keys(selectedEmbeddingsProvider).length > 0 && Object.keys(selectedVectorStoreProvider).length > 0 && (
|
||||
<Button
|
||||
variant='contained'
|
||||
sx={{
|
||||
borderRadius: 2,
|
||||
height: '100%',
|
||||
backgroundImage: `linear-gradient(to right, #13547a, #2f9e91)`,
|
||||
'&:hover': {
|
||||
backgroundImage: `linear-gradient(to right, #0b3d5b, #1a8377)`
|
||||
}
|
||||
}}
|
||||
startIcon={<IconRowInsertTop />}
|
||||
onClick={() => tryAndInsertIntoStore()}
|
||||
>
|
||||
Upsert
|
||||
</Button>
|
||||
)}
|
||||
<IconButton onClick={showUpsertHistoryDrawer} size='small' color='inherit' title='Upsert History'>
|
||||
<IconClock />
|
||||
</IconButton>
|
||||
</ViewHeader>
|
||||
<Steps />
|
||||
<Grid container spacing={1}>
|
||||
<Grid item xs={12} sm={4} md={4}>
|
||||
{Object.keys(selectedEmbeddingsProvider).length === 0 ? (
|
||||
<Button
|
||||
onClick={showEmbeddingsList}
|
||||
fullWidth={true}
|
||||
startIcon={<Embeddings style={{ background: 'transparent', height: 32, width: 32 }} />}
|
||||
sx={{
|
||||
color: customization?.isDarkMode ? 'white' : 'inherit',
|
||||
borderRadius: '10px',
|
||||
minHeight: '200px',
|
||||
boxShadow: '0 2px 14px 0 rgb(32 40 45 / 20%)',
|
||||
backgroundImage: customization?.isDarkMode
|
||||
? `linear-gradient(to right, #e654bc, #4b86e7)`
|
||||
: `linear-gradient(to right, #fadef2, #cfdcf1)`,
|
||||
'&:hover': {
|
||||
backgroundImage: customization?.isDarkMode
|
||||
? `linear-gradient(to right, #de32ac, #2d73e7)`
|
||||
: `linear-gradient(to right, #f6c2e7, #b4cbf1)`
|
||||
}
|
||||
}}
|
||||
>
|
||||
Select Embeddings
|
||||
</Button>
|
||||
) : (
|
||||
<Box>
|
||||
<Grid container spacing='2'>
|
||||
<Grid item xs={12} md={12} lg={12} sm={12}>
|
||||
<div
|
||||
style={{
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
paddingRight: 15
|
||||
}}
|
||||
>
|
||||
<Box
|
||||
sx={{
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
flexDirection: 'row',
|
||||
p: 1
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
width: 40,
|
||||
height: 40,
|
||||
borderRadius: '50%',
|
||||
backgroundColor: 'white',
|
||||
display: 'flex',
|
||||
boxShadow: '0 2px 14px 0 rgb(32 40 45 / 25%)'
|
||||
}}
|
||||
>
|
||||
{selectedEmbeddingsProvider.label ? (
|
||||
<img
|
||||
style={{
|
||||
width: '100%',
|
||||
height: '100%',
|
||||
padding: 7,
|
||||
borderRadius: '50%',
|
||||
objectFit: 'contain'
|
||||
}}
|
||||
alt={selectedEmbeddingsProvider.label ?? 'embeddings'}
|
||||
src={`${baseURL}/api/v1/node-icon/${selectedEmbeddingsProvider?.name}`}
|
||||
/>
|
||||
) : (
|
||||
<Embeddings color='black' />
|
||||
)}
|
||||
</div>
|
||||
<Typography sx={{ ml: 2 }} variant='h3'>
|
||||
{selectedEmbeddingsProvider.label}
|
||||
</Typography>
|
||||
<div style={{ flex: 1 }}></div>
|
||||
<div
|
||||
style={{
|
||||
display: 'flex',
|
||||
alignContent: 'center',
|
||||
flexDirection: 'row'
|
||||
}}
|
||||
>
|
||||
{Object.keys(selectedEmbeddingsProvider).length > 0 && (
|
||||
<>
|
||||
<IconButton
|
||||
variant='outlined'
|
||||
sx={{ ml: 1 }}
|
||||
color='secondary'
|
||||
onClick={showEmbeddingsList}
|
||||
>
|
||||
<IconEditCircle />
|
||||
</IconButton>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</Box>
|
||||
{selectedEmbeddingsProvider &&
|
||||
Object.keys(selectedEmbeddingsProvider).length > 0 &&
|
||||
(selectedEmbeddingsProvider.inputParams ?? [])
|
||||
.filter((inputParam) => !inputParam.hidden)
|
||||
.map((inputParam, index) => (
|
||||
<DocStoreInputHandler
|
||||
key={index}
|
||||
data={selectedEmbeddingsProvider}
|
||||
inputParam={inputParam}
|
||||
isAdditionalParams={inputParam.additionalParams}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</Grid>
|
||||
</Grid>
|
||||
</Box>
|
||||
)}
|
||||
</Grid>
|
||||
<Grid item xs={12} sm={4} md={4}>
|
||||
{Object.keys(selectedVectorStoreProvider).length === 0 ? (
|
||||
<Button
|
||||
onClick={showVectorStoreList}
|
||||
fullWidth={true}
|
||||
startIcon={<Storage style={{ background: 'transparent', height: 32, width: 32 }} />}
|
||||
sx={{
|
||||
color: customization?.isDarkMode ? 'white' : 'inherit',
|
||||
borderRadius: '10px',
|
||||
minHeight: '200px',
|
||||
opacity: isVectorStoreDisabled() ? 0.7 : 1,
|
||||
boxShadow: isVectorStoreDisabled() ? 'none' : '0 2px 14px 0 rgb(32 40 45 / 20%)',
|
||||
backgroundImage: customization?.isDarkMode
|
||||
? `linear-gradient(to right, #4d8ef1, #f1de5c)`
|
||||
: `linear-gradient(to right, #b9d0f4, #fef9d7)`,
|
||||
'&:hover': {
|
||||
backgroundImage: customization?.isDarkMode
|
||||
? `linear-gradient(to right, #2576f2, #f0d72e)`
|
||||
: `linear-gradient(to right, #9cbdf2, #fcf3b6)`
|
||||
}
|
||||
}}
|
||||
disabled={isVectorStoreDisabled()}
|
||||
>
|
||||
Select Vector Store
|
||||
</Button>
|
||||
) : (
|
||||
<Box>
|
||||
<Grid container spacing='2'>
|
||||
<Grid item xs={12} md={12} lg={12} sm={12}>
|
||||
<div
|
||||
style={{
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
paddingRight: 15
|
||||
}}
|
||||
>
|
||||
<Box
|
||||
sx={{
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
flexDirection: 'row',
|
||||
p: 1
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
width: 40,
|
||||
height: 40,
|
||||
borderRadius: '50%',
|
||||
backgroundColor: 'white',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
boxShadow: '0 2px 14px 0 rgb(32 40 45 / 25%)'
|
||||
}}
|
||||
>
|
||||
{selectedVectorStoreProvider.label ? (
|
||||
<img
|
||||
style={{
|
||||
width: '100%',
|
||||
height: '100%',
|
||||
padding: 7,
|
||||
borderRadius: '50%',
|
||||
objectFit: 'contain'
|
||||
}}
|
||||
alt={selectedVectorStoreProvider.label ?? 'embeddings'}
|
||||
src={`${baseURL}/api/v1/node-icon/${selectedVectorStoreProvider?.name}`}
|
||||
/>
|
||||
) : (
|
||||
<Embeddings color='black' />
|
||||
)}
|
||||
</div>
|
||||
<Typography sx={{ ml: 2 }} variant='h3'>
|
||||
{selectedVectorStoreProvider.label}
|
||||
</Typography>
|
||||
<div style={{ flex: 1 }}></div>
|
||||
<div
|
||||
style={{
|
||||
display: 'flex',
|
||||
alignContent: 'center',
|
||||
flexDirection: 'row'
|
||||
}}
|
||||
>
|
||||
{Object.keys(selectedVectorStoreProvider).length > 0 && (
|
||||
<>
|
||||
<IconButton
|
||||
variant='outlined'
|
||||
sx={{ ml: 1 }}
|
||||
color='secondary'
|
||||
onClick={showVectorStoreList}
|
||||
>
|
||||
<IconEditCircle />
|
||||
</IconButton>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</Box>
|
||||
{selectedVectorStoreProvider &&
|
||||
Object.keys(selectedVectorStoreProvider).length > 0 &&
|
||||
(selectedVectorStoreProvider.inputParams ?? [])
|
||||
.filter((inputParam) => !inputParam.hidden)
|
||||
.map((inputParam, index) => (
|
||||
<DocStoreInputHandler
|
||||
key={index}
|
||||
data={selectedVectorStoreProvider}
|
||||
inputParam={inputParam}
|
||||
isAdditionalParams={inputParam.additionalParams}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</Grid>
|
||||
</Grid>
|
||||
</Box>
|
||||
)}
|
||||
</Grid>
|
||||
<Grid item xs={12} sm={4} md={4}>
|
||||
{Object.keys(selectedRecordManagerProvider).length === 0 ? (
|
||||
<Button
|
||||
onClick={showRecordManagerList}
|
||||
fullWidth={true}
|
||||
startIcon={
|
||||
isRecordManagerUnavailable ? (
|
||||
<></>
|
||||
) : (
|
||||
<DynamicFeed style={{ background: 'transparent', height: 32, width: 32 }} />
|
||||
)
|
||||
}
|
||||
sx={{
|
||||
color: customization?.isDarkMode ? 'white' : 'inherit',
|
||||
borderRadius: '10px',
|
||||
minHeight: '200px',
|
||||
opacity: isRecordManagerDisabled() ? 0.7 : 1,
|
||||
boxShadow: isRecordManagerDisabled() ? 'none' : '0 2px 14px 0 rgb(32 40 45 / 20%)',
|
||||
backgroundImage: customization?.isDarkMode
|
||||
? `linear-gradient(to right, #f5db3f, #42daa7)`
|
||||
: `linear-gradient(to right, #f9f1c0, #c7f1e3)`,
|
||||
'&:hover': {
|
||||
backgroundImage: customization?.isDarkMode
|
||||
? `linear-gradient(to right, #d9c238, #3dc295)`
|
||||
: `linear-gradient(to right, #f6e99b, #a0f2d7)`
|
||||
}
|
||||
}}
|
||||
disabled={isRecordManagerDisabled()}
|
||||
>
|
||||
{isRecordManagerUnavailable
|
||||
? 'Record Manager is not applicable for selected Vector Store'
|
||||
: 'Select Record Manager'}
|
||||
</Button>
|
||||
) : (
|
||||
<Box>
|
||||
<Grid container spacing='2'>
|
||||
<Grid item xs={12} md={12} lg={12} sm={12}>
|
||||
<div
|
||||
style={{
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
paddingRight: 15
|
||||
}}
|
||||
>
|
||||
<Box
|
||||
sx={{
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
flexDirection: 'row',
|
||||
p: 1
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
width: 40,
|
||||
height: 40,
|
||||
borderRadius: '50%',
|
||||
backgroundColor: 'white',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
boxShadow: '0 2px 14px 0 rgb(32 40 45 / 25%)'
|
||||
}}
|
||||
>
|
||||
{selectedRecordManagerProvider.label ? (
|
||||
<img
|
||||
style={{
|
||||
width: '100%',
|
||||
height: '100%',
|
||||
padding: 7,
|
||||
borderRadius: '50%',
|
||||
objectFit: 'contain'
|
||||
}}
|
||||
alt={selectedRecordManagerProvider.label ?? 'embeddings'}
|
||||
src={`${baseURL}/api/v1/node-icon/${selectedRecordManagerProvider?.name}`}
|
||||
/>
|
||||
) : (
|
||||
<Embeddings color='black' />
|
||||
)}
|
||||
</div>
|
||||
<Typography sx={{ ml: 2 }} variant='h3'>
|
||||
{selectedRecordManagerProvider.label}
|
||||
</Typography>
|
||||
<div style={{ flex: 1 }}></div>
|
||||
<div
|
||||
style={{
|
||||
display: 'flex',
|
||||
alignContent: 'center',
|
||||
flexDirection: 'row'
|
||||
}}
|
||||
>
|
||||
{Object.keys(selectedRecordManagerProvider).length > 0 && (
|
||||
<>
|
||||
<IconButton
|
||||
variant='outlined'
|
||||
sx={{ ml: 1 }}
|
||||
color='secondary'
|
||||
onClick={showRecordManagerList}
|
||||
>
|
||||
<IconEditCircle />
|
||||
</IconButton>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</Box>
|
||||
{selectedRecordManagerProvider &&
|
||||
Object.keys(selectedRecordManagerProvider).length > 0 &&
|
||||
(selectedRecordManagerProvider.inputParams ?? [])
|
||||
.filter((inputParam) => !inputParam.hidden)
|
||||
.map((inputParam, index) => (
|
||||
<>
|
||||
<DocStoreInputHandler
|
||||
key={index}
|
||||
data={selectedRecordManagerProvider}
|
||||
inputParam={inputParam}
|
||||
isAdditionalParams={inputParam.additionalParams}
|
||||
/>
|
||||
</>
|
||||
))}
|
||||
</div>
|
||||
</Grid>
|
||||
</Grid>
|
||||
</Box>
|
||||
)}
|
||||
</Grid>
|
||||
</Grid>
|
||||
</Stack>
|
||||
)}
|
||||
</MainCard>
|
||||
|
||||
{showEmbeddingsListDialog && (
|
||||
<ComponentsListDialog
|
||||
show={showEmbeddingsListDialog}
|
||||
dialogProps={dialogProps}
|
||||
onCancel={() => setShowEmbeddingsListDialog(false)}
|
||||
apiCall={documentsApi.getEmbeddingProviders}
|
||||
onSelected={onEmbeddingsSelected}
|
||||
/>
|
||||
)}
|
||||
{showVectorStoreListDialog && (
|
||||
<ComponentsListDialog
|
||||
show={showVectorStoreListDialog}
|
||||
dialogProps={dialogProps}
|
||||
onCancel={() => setShowVectorStoreListDialog(false)}
|
||||
apiCall={documentsApi.getVectorStoreProviders}
|
||||
onSelected={onVectorStoreSelected}
|
||||
/>
|
||||
)}
|
||||
{showRecordManagerListDialog && (
|
||||
<ComponentsListDialog
|
||||
show={showRecordManagerListDialog}
|
||||
dialogProps={dialogProps}
|
||||
onCancel={() => setShowRecordManagerListDialog(false)}
|
||||
apiCall={documentsApi.getRecordManagerProviders}
|
||||
onSelected={onRecordManagerSelected}
|
||||
/>
|
||||
)}
|
||||
{showUpsertHistoryDialog && (
|
||||
<UpsertResultDialog
|
||||
show={showUpsertHistoryDialog}
|
||||
dialogProps={upsertResultDialogProps}
|
||||
onCancel={() => {
|
||||
setShowUpsertHistoryDialog(false)
|
||||
}}
|
||||
onGoToRetrievalQuery={() => navigate('/document-stores/query/' + storeId)}
|
||||
></UpsertResultDialog>
|
||||
)}
|
||||
{showUpsertHistorySideDrawer && (
|
||||
<UpsertHistorySideDrawer
|
||||
show={showUpsertHistorySideDrawer}
|
||||
dialogProps={upsertHistoryDrawerDialogProps}
|
||||
onClickFunction={() => setShowUpsertHistorySideDrawer(false)}
|
||||
onSelectHistoryDetails={onSelectHistoryDetails}
|
||||
/>
|
||||
)}
|
||||
{showUpsertHistoryDetailsDialog && (
|
||||
<UpsertHistoryDetailsDialog
|
||||
show={showUpsertHistoryDetailsDialog}
|
||||
dialogProps={upsertDetailsDialogProps}
|
||||
onCancel={() => setShowUpsertHistoryDetailsDialog(false)}
|
||||
/>
|
||||
)}
|
||||
<ConfirmDialog />
|
||||
{loading && <BackdropLoader open={loading} />}
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
export default VectorStoreConfigure
|
||||
@@ -0,0 +1,483 @@
|
||||
import { useEffect, useState, useRef } from 'react'
|
||||
import { useDispatch, useSelector } from 'react-redux'
|
||||
import { useNavigate } from 'react-router-dom'
|
||||
import ReactJson from 'flowise-react-json-view'
|
||||
import { cloneDeep } from 'lodash'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
|
||||
// material-ui
|
||||
import { Box, Card, Grid, Stack, Typography, OutlinedInput, IconButton, Button } from '@mui/material'
|
||||
import Embeddings from '@mui/icons-material/DynamicFeed'
|
||||
import { useTheme, styled } from '@mui/material/styles'
|
||||
import CardContent from '@mui/material/CardContent'
|
||||
import chunks_emptySVG from '@/assets/images/chunks_empty.svg'
|
||||
import { IconSearch, IconFileStack, IconDeviceFloppy, IconX } from '@tabler/icons-react'
|
||||
|
||||
// project imports
|
||||
import MainCard from '@/ui-component/cards/MainCard'
|
||||
import { BackdropLoader } from '@/ui-component/loading/BackdropLoader'
|
||||
import ConfirmDialog from '@/ui-component/dialog/ConfirmDialog'
|
||||
import ExpandedChunkDialog from './ExpandedChunkDialog'
|
||||
import ViewHeader from '@/layout/MainLayout/ViewHeader'
|
||||
import DocStoreInputHandler from '@/views/docstore/DocStoreInputHandler'
|
||||
|
||||
// API
|
||||
import documentsApi from '@/api/documentstore'
|
||||
import nodesApi from '@/api/nodes'
|
||||
|
||||
// Hooks
|
||||
import useApi from '@/hooks/useApi'
|
||||
import useNotifier from '@/utils/useNotifier'
|
||||
import { baseURL } from '@/store/constant'
|
||||
import { initNode } from '@/utils/genericHelper'
|
||||
import { closeSnackbar as closeSnackbarAction, enqueueSnackbar as enqueueSnackbarAction } from '@/store/actions'
|
||||
|
||||
const CardWrapper = styled(MainCard)(({ theme }) => ({
|
||||
background: theme.palette.card.main,
|
||||
color: theme.darkTextPrimary,
|
||||
overflow: 'auto',
|
||||
position: 'relative',
|
||||
boxShadow: '0 2px 14px 0 rgb(32 40 45 / 8%)',
|
||||
cursor: 'pointer',
|
||||
'&:hover': {
|
||||
background: theme.palette.card.hover,
|
||||
boxShadow: '0 2px 14px 0 rgb(32 40 45 / 20%)'
|
||||
},
|
||||
maxHeight: '250px',
|
||||
minHeight: '250px',
|
||||
maxWidth: '100%',
|
||||
overflowWrap: 'break-word',
|
||||
whiteSpace: 'pre-line',
|
||||
padding: 1
|
||||
}))
|
||||
|
||||
const VectorStoreQuery = () => {
|
||||
const customization = useSelector((state) => state.customization)
|
||||
const navigate = useNavigate()
|
||||
const theme = useTheme()
|
||||
const dispatch = useDispatch()
|
||||
const inputRef = useRef(null)
|
||||
|
||||
useNotifier()
|
||||
|
||||
const enqueueSnackbar = (...args) => dispatch(enqueueSnackbarAction(...args))
|
||||
const closeSnackbar = (...args) => dispatch(closeSnackbarAction(...args))
|
||||
|
||||
const URLpath = document.location.pathname.toString().split('/')
|
||||
const storeId = URLpath[URLpath.length - 1] === 'document-stores' ? '' : URLpath[URLpath.length - 1]
|
||||
|
||||
const [documentChunks, setDocumentChunks] = useState([])
|
||||
const [loading, setLoading] = useState(false)
|
||||
const [showExpandedChunkDialog, setShowExpandedChunkDialog] = useState(false)
|
||||
const [expandedChunkDialogProps, setExpandedChunkDialogProps] = useState({})
|
||||
const [documentStore, setDocumentStore] = useState({})
|
||||
const [query, setQuery] = useState('')
|
||||
|
||||
const [timeTaken, setTimeTaken] = useState(-1)
|
||||
const [retrievalError, setRetrievalError] = useState(undefined)
|
||||
|
||||
const getSpecificDocumentStoreApi = useApi(documentsApi.getSpecificDocumentStore)
|
||||
const queryVectorStoreApi = useApi(documentsApi.queryVectorStore)
|
||||
|
||||
const getVectorStoreNodeDetailsApi = useApi(nodesApi.getSpecificNode)
|
||||
const [selectedVectorStoreProvider, setSelectedVectorStoreProvider] = useState({})
|
||||
|
||||
const chunkSelected = (chunkId, selectedChunkNumber) => {
|
||||
const selectedChunk = documentChunks.find((chunk) => chunk.id === chunkId)
|
||||
const dialogProps = {
|
||||
data: {
|
||||
selectedChunk,
|
||||
selectedChunkNumber
|
||||
}
|
||||
}
|
||||
setExpandedChunkDialogProps(dialogProps)
|
||||
setShowExpandedChunkDialog(true)
|
||||
}
|
||||
|
||||
const handleEnter = (e) => {
|
||||
// Check if IME composition is in progress
|
||||
const isIMEComposition = e.isComposing || e.keyCode === 229
|
||||
if (e.key === 'Enter' && query && !isIMEComposition) {
|
||||
if (!e.shiftKey && query) {
|
||||
if (inputRef.current) {
|
||||
inputRef.current.blur()
|
||||
}
|
||||
doQuery()
|
||||
}
|
||||
} else if (e.key === 'Enter') {
|
||||
e.preventDefault()
|
||||
}
|
||||
}
|
||||
|
||||
const doQuery = () => {
|
||||
setLoading(true)
|
||||
const data = {
|
||||
query: query,
|
||||
storeId: storeId,
|
||||
inputs: selectedVectorStoreProvider.inputs
|
||||
}
|
||||
queryVectorStoreApi.request(data)
|
||||
}
|
||||
|
||||
const saveConfig = async () => {
|
||||
setLoading(true)
|
||||
const data = {
|
||||
storeId: storeId
|
||||
}
|
||||
|
||||
if (selectedVectorStoreProvider.inputs) {
|
||||
data.vectorStoreConfig = {}
|
||||
data.vectorStoreName = selectedVectorStoreProvider.name
|
||||
Object.keys(selectedVectorStoreProvider.inputs).map((key) => {
|
||||
if (key === 'FLOWISE_CREDENTIAL_ID') {
|
||||
data.vectorStoreConfig['credential'] = selectedVectorStoreProvider.inputs[key]
|
||||
} else {
|
||||
data.vectorStoreConfig[key] = selectedVectorStoreProvider.inputs[key]
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
const updateResp = await documentsApi.updateVectorStoreConfig(data)
|
||||
setLoading(false)
|
||||
if (updateResp.data) {
|
||||
enqueueSnackbar({
|
||||
message: 'Vector Store Config Successfully Updated',
|
||||
options: {
|
||||
key: new Date().getTime() + Math.random(),
|
||||
variant: 'success',
|
||||
action: (key) => (
|
||||
<Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
|
||||
<IconX />
|
||||
</Button>
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
} catch (error) {
|
||||
setLoading(false)
|
||||
const errorData = error.response?.data || `${error.response?.status}: ${error.response?.statusText}`
|
||||
enqueueSnackbar({
|
||||
message: `Failed to update vector store config: ${errorData}`,
|
||||
options: {
|
||||
key: new Date().getTime() + Math.random(),
|
||||
variant: 'error',
|
||||
persist: true,
|
||||
action: (key) => (
|
||||
<Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
|
||||
<IconX />
|
||||
</Button>
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
if (queryVectorStoreApi.data) {
|
||||
setDocumentChunks(queryVectorStoreApi.data.docs)
|
||||
setTimeTaken(queryVectorStoreApi.data.timeTaken)
|
||||
setRetrievalError(undefined)
|
||||
setLoading(false)
|
||||
if (inputRef.current) {
|
||||
inputRef.current.focus()
|
||||
}
|
||||
}
|
||||
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [queryVectorStoreApi.data])
|
||||
|
||||
useEffect(() => {
|
||||
if (queryVectorStoreApi.error) {
|
||||
if (queryVectorStoreApi.error.response?.data?.message) {
|
||||
const message = queryVectorStoreApi.error.response.data.message
|
||||
// remove the text 'documentStoreServices.queryVectorStore - ' from the error message to make it readable
|
||||
setRetrievalError(message.replace('documentStoreServices.queryVectorStore - ', ''))
|
||||
setDocumentChunks([])
|
||||
setTimeTaken(-1)
|
||||
}
|
||||
setLoading(false)
|
||||
}
|
||||
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [queryVectorStoreApi.error])
|
||||
|
||||
useEffect(() => {
|
||||
if (getVectorStoreNodeDetailsApi.data) {
|
||||
const node = getVectorStoreNodeDetailsApi.data
|
||||
fetchVectorStoreDetails(node)
|
||||
}
|
||||
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [getVectorStoreNodeDetailsApi.data])
|
||||
|
||||
const fetchVectorStoreDetails = (component) => {
|
||||
const nodeData = cloneDeep(initNode(component, uuidv4()))
|
||||
if (documentStore.vectorStoreConfig) {
|
||||
nodeData.inputs = documentStore.vectorStoreConfig.config
|
||||
nodeData.credential = documentStore.vectorStoreConfig.config.credential
|
||||
}
|
||||
setSelectedVectorStoreProvider(nodeData)
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
getSpecificDocumentStoreApi.request(storeId)
|
||||
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
if (getSpecificDocumentStoreApi.data) {
|
||||
setDocumentStore(getSpecificDocumentStoreApi.data)
|
||||
const vectorStoreConfig = getSpecificDocumentStoreApi.data.vectorStoreConfig
|
||||
if (vectorStoreConfig) {
|
||||
getVectorStoreNodeDetailsApi.request(vectorStoreConfig.name)
|
||||
}
|
||||
setLoading(false)
|
||||
}
|
||||
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [getSpecificDocumentStoreApi.data])
|
||||
|
||||
return (
|
||||
<>
|
||||
<MainCard style={{ position: 'relative' }}>
|
||||
<Stack flexDirection='column' sx={{ gap: 1 }}>
|
||||
<ViewHeader
|
||||
isBackButton={true}
|
||||
search={false}
|
||||
title={documentStore?.name || 'Document Store'}
|
||||
description='Retrieval Playground - Test your vector store retrieval settings'
|
||||
onBack={() => navigate(-1)}
|
||||
>
|
||||
<Button
|
||||
variant='outlined'
|
||||
color='secondary'
|
||||
sx={{ borderRadius: 2, height: '100%' }}
|
||||
startIcon={<IconDeviceFloppy />}
|
||||
onClick={saveConfig}
|
||||
>
|
||||
Save Config
|
||||
</Button>
|
||||
</ViewHeader>
|
||||
<div style={{ width: '100%' }}></div>
|
||||
<div>
|
||||
<Grid container spacing={2}>
|
||||
<Grid sx={{ ml: 1, mr: 1 }} item xs={12} sm={12} md={12} lg={12}>
|
||||
<Box>
|
||||
<div style={{ display: 'flex', flexDirection: 'row' }}>
|
||||
<Typography variant='overline'>
|
||||
Enter your Query<span style={{ color: 'red' }}> *</span>
|
||||
</Typography>
|
||||
|
||||
<div style={{ flexGrow: 1 }}></div>
|
||||
</div>
|
||||
<OutlinedInput
|
||||
size='small'
|
||||
multiline={true}
|
||||
rows={4}
|
||||
sx={{ mt: 1 }}
|
||||
type='string'
|
||||
fullWidth
|
||||
inputRef={inputRef}
|
||||
onChange={(e) => setQuery(e.target.value)}
|
||||
onKeyDown={handleEnter}
|
||||
value={query ?? ''}
|
||||
endAdornment={
|
||||
<IconButton variant='contained' onClick={doQuery}>
|
||||
<IconSearch />
|
||||
</IconButton>
|
||||
}
|
||||
/>
|
||||
</Box>
|
||||
</Grid>
|
||||
<Grid sx={{ ml: 1, mr: 1, mt: 1 }} container spacing={1}>
|
||||
<Grid item xs={12} sm={4} md={4}>
|
||||
<Box>
|
||||
<Grid container spacing='2'>
|
||||
<Grid item xs={12} md={12} lg={12} sm={12}>
|
||||
<div
|
||||
style={{
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
paddingRight: 15,
|
||||
paddingTop: 5
|
||||
}}
|
||||
>
|
||||
<Box
|
||||
sx={{
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
flexDirection: 'row',
|
||||
p: 1
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
width: 40,
|
||||
height: 40,
|
||||
borderRadius: '50%',
|
||||
backgroundColor: 'white',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
boxShadow: '0 2px 14px 0 rgb(32 40 45 / 25%)'
|
||||
}}
|
||||
>
|
||||
{selectedVectorStoreProvider.label ? (
|
||||
<img
|
||||
style={{
|
||||
width: '100%',
|
||||
height: '100%',
|
||||
padding: 7,
|
||||
borderRadius: '50%',
|
||||
objectFit: 'contain'
|
||||
}}
|
||||
alt={selectedVectorStoreProvider.label ?? 'embeddings'}
|
||||
src={`${baseURL}/api/v1/node-icon/${selectedVectorStoreProvider?.name}`}
|
||||
/>
|
||||
) : (
|
||||
<Embeddings color='black' />
|
||||
)}
|
||||
</div>
|
||||
<Typography sx={{ ml: 2 }} variant='h3'>
|
||||
{selectedVectorStoreProvider.label}
|
||||
</Typography>
|
||||
<div style={{ flex: 1 }}></div>
|
||||
</Box>
|
||||
{selectedVectorStoreProvider &&
|
||||
Object.keys(selectedVectorStoreProvider).length > 0 &&
|
||||
(selectedVectorStoreProvider.inputParams ?? [])
|
||||
.filter((inputParam) => !inputParam.hidden)
|
||||
.map((inputParam, index) => (
|
||||
<DocStoreInputHandler
|
||||
key={index}
|
||||
data={selectedVectorStoreProvider}
|
||||
inputParam={inputParam}
|
||||
isAdditionalParams={inputParam.additionalParams}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</Grid>
|
||||
</Grid>
|
||||
</Box>
|
||||
</Grid>
|
||||
<Grid item xs={12} sm={8} md={8}>
|
||||
<Box
|
||||
sx={{
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
flexDirection: 'row',
|
||||
p: 1,
|
||||
paddingTop: 2,
|
||||
marginBottom: 4
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
width: 40,
|
||||
height: 40,
|
||||
borderRadius: '50%',
|
||||
backgroundColor: 'white',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
boxShadow: '0 2px 14px 0 rgb(32 40 45 / 25%)'
|
||||
}}
|
||||
>
|
||||
<IconFileStack
|
||||
style={{
|
||||
width: '100%',
|
||||
height: '100%',
|
||||
padding: 7,
|
||||
borderRadius: '50%',
|
||||
objectFit: 'contain'
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
<Typography sx={{ ml: 2 }} variant='h3'>
|
||||
Retrieved Documents
|
||||
{timeTaken > -1 && (
|
||||
<Typography variant='body2' sx={{ color: 'gray' }}>
|
||||
Count: {documentChunks.length}. Time taken: {timeTaken} millis.
|
||||
</Typography>
|
||||
)}
|
||||
{retrievalError && (
|
||||
<Typography variant='body2' sx={{ color: 'gray' }}>
|
||||
{retrievalError}
|
||||
</Typography>
|
||||
)}
|
||||
</Typography>
|
||||
<div style={{ flex: 1 }}></div>
|
||||
</Box>
|
||||
{!documentChunks.length && (
|
||||
<div
|
||||
style={{
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
width: '100%'
|
||||
}}
|
||||
>
|
||||
<Box sx={{ mt: 5, p: 2, height: 'auto' }}>
|
||||
<img
|
||||
style={{ objectFit: 'cover', height: '16vh', width: 'auto' }}
|
||||
src={chunks_emptySVG}
|
||||
alt='chunks_emptySVG'
|
||||
/>
|
||||
</Box>
|
||||
<div>No Documents Retrieved</div>
|
||||
</div>
|
||||
)}
|
||||
<Grid container spacing={2}>
|
||||
{documentChunks?.length > 0 &&
|
||||
documentChunks.map((row, index) => (
|
||||
<Grid item lg={6} md={6} sm={6} xs={6} key={index}>
|
||||
<CardWrapper
|
||||
content={false}
|
||||
onClick={() => chunkSelected(row.id, row.chunkNo)}
|
||||
sx={{ border: 1, borderColor: theme.palette.grey[900] + 25, borderRadius: 2 }}
|
||||
>
|
||||
<Card>
|
||||
<CardContent sx={{ p: 2 }}>
|
||||
<Typography sx={{ wordWrap: 'break-word', mb: 1 }} variant='h5'>
|
||||
{`#${row.chunkNo}. Characters: ${row.pageContent.length}`}
|
||||
</Typography>
|
||||
<Typography sx={{ wordWrap: 'break-word' }} variant='body2'>
|
||||
{row.pageContent}
|
||||
</Typography>
|
||||
<ReactJson
|
||||
theme={customization.isDarkMode ? 'ocean' : 'rjv-default'}
|
||||
style={{ paddingTop: 10 }}
|
||||
src={row.metadata || {}}
|
||||
name={null}
|
||||
quotesOnKeys={false}
|
||||
enableClipboard={false}
|
||||
displayDataTypes={false}
|
||||
collapsed={true}
|
||||
/>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</CardWrapper>
|
||||
</Grid>
|
||||
))}
|
||||
</Grid>
|
||||
</Grid>
|
||||
</Grid>
|
||||
</Grid>
|
||||
</div>
|
||||
</Stack>
|
||||
</MainCard>
|
||||
<ConfirmDialog />
|
||||
<ExpandedChunkDialog
|
||||
show={showExpandedChunkDialog}
|
||||
dialogProps={expandedChunkDialogProps}
|
||||
onCancel={() => setShowExpandedChunkDialog(false)}
|
||||
isReadOnly={true}
|
||||
></ExpandedChunkDialog>
|
||||
{loading && <BackdropLoader open={loading} />}
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
export default VectorStoreQuery
|
||||
@@ -6,8 +6,9 @@ import ReactJson from 'flowise-react-json-view'
|
||||
import { Typography, Card, CardContent, Button, Dialog, DialogActions, DialogContent, DialogTitle } from '@mui/material'
|
||||
import StatsCard from '@/ui-component/cards/StatsCard'
|
||||
import { HIDE_CANVAS_DIALOG, SHOW_CANVAS_DIALOG } from '@/store/actions'
|
||||
import { IconZoomScan } from '@tabler/icons-react'
|
||||
|
||||
const UpsertResultDialog = ({ show, dialogProps, onCancel }) => {
|
||||
const UpsertResultDialog = ({ show, dialogProps, onCancel, onGoToRetrievalQuery }) => {
|
||||
const portalElement = document.getElementById('portal')
|
||||
const dispatch = useDispatch()
|
||||
const customization = useSelector((state) => state.customization)
|
||||
@@ -76,7 +77,31 @@ const UpsertResultDialog = ({ show, dialogProps, onCancel }) => {
|
||||
</>
|
||||
</DialogContent>
|
||||
<DialogActions>
|
||||
<Button onClick={onCancel}>Close</Button>
|
||||
{dialogProps.goToRetrievalQuery && (
|
||||
<div style={{ display: 'flex', flexDirection: 'column', width: '100%', marginLeft: '15px', marginRight: '15px' }}>
|
||||
<Button
|
||||
variant='contained'
|
||||
fullWidth
|
||||
sx={{
|
||||
borderRadius: 2,
|
||||
height: '100%',
|
||||
backgroundImage: `linear-gradient(to right, #3f5efb, #fc466b)`,
|
||||
'&:hover': {
|
||||
backgroundImage: `linear-gradient(to right, #2b4efb, #fe2752)`
|
||||
},
|
||||
mb: 2
|
||||
}}
|
||||
startIcon={<IconZoomScan />}
|
||||
onClick={onGoToRetrievalQuery}
|
||||
>
|
||||
Test Retrieval
|
||||
</Button>
|
||||
<Button fullWidth onClick={onCancel}>
|
||||
Close
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
{!dialogProps.goToRetrievalQuery && <Button onClick={onCancel}>Close</Button>}
|
||||
</DialogActions>
|
||||
</Dialog>
|
||||
) : null
|
||||
@@ -88,7 +113,7 @@ UpsertResultDialog.propTypes = {
|
||||
show: PropTypes.bool,
|
||||
dialogProps: PropTypes.object,
|
||||
onCancel: PropTypes.func,
|
||||
onSave: PropTypes.func
|
||||
onGoToRetrievalQuery: PropTypes.func
|
||||
}
|
||||
|
||||
export default UpsertResultDialog
|
||||
|
||||
Reference in New Issue
Block a user