From 66acd0c0006a949e7e88418b3cc9beebccb06507 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Sun, 25 Aug 2024 13:22:48 +0100 Subject: [PATCH] Feature/add ability to upload file from chat (#3059) add ability to upload file from chat --- .../nodes/agents/CSVAgent/CSVAgent.ts | 2 + .../chatmodels/AWSBedrock/AWSChatBedrock.ts | 2 +- .../AzureChatOpenAI/AzureChatOpenAI.ts | 2 +- .../chatmodels/ChatAnthropic/ChatAnthropic.ts | 2 +- .../ChatAnthropic/FlowiseChatAnthropic.ts | 7 +- .../ChatGoogleGenerativeAI.ts | 2 +- .../nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts | 2 +- .../nodes/documentloaders/Csv/Csv.ts | 2 + .../nodes/documentloaders/Docx/Docx.ts | 2 + .../nodes/documentloaders/File/File.ts | 299 ++++++++++++ .../nodes/documentloaders/File/file.svg | 1 + .../nodes/documentloaders/Folder/Folder.ts | 19 +- .../nodes/documentloaders/Json/Json.ts | 2 + .../documentloaders/Jsonlines/Jsonlines.ts | 2 + .../nodes/documentloaders/Pdf/Pdf.ts | 2 + .../nodes/documentloaders/Text/Text.ts | 2 + .../Unstructured/UnstructuredFile.ts | 2 + .../multiagents/Supervisor/Supervisor.ts | 8 +- .../nodes/vectorstores/Milvus/Milvus.ts | 30 +- .../nodes/vectorstores/Pinecone/Pinecone.ts | 31 +- .../nodes/vectorstores/Postgres/Postgres.ts | 40 +- .../nodes/vectorstores/Qdrant/Qdrant.ts | 41 +- .../nodes/vectorstores/Upstash/Upstash.ts | 27 +- .../nodes/vectorstores/Vectara/Vectara.ts | 2 + .../nodes/vectorstores/VectorStoreUtils.ts | 16 + packages/components/src/utils.ts | 66 +++ .../marketplaces/chatflows/SQL Prompt.json | 30 +- packages/server/src/ChatflowPool.ts | 10 +- packages/server/src/Interface.ts | 1 + packages/server/src/routes/vectors/index.ts | 2 +- packages/server/src/utils/buildChatflow.ts | 17 +- packages/server/src/utils/getUploadsConfig.ts | 115 +++-- packages/server/src/utils/index.ts | 35 +- packages/server/src/utils/upsertVector.ts | 21 +- packages/ui/src/api/vectorstore.js | 5 + .../dialog/ViewMessagesDialog.jsx | 87 ++-- .../ui/src/views/chatmessage/ChatMessage.jsx | 434 +++++++++++++----- 37 files changed, 1111 insertions(+), 259 deletions(-) create mode 100644 packages/components/nodes/documentloaders/File/File.ts create mode 100644 packages/components/nodes/documentloaders/File/file.svg diff --git a/packages/components/nodes/agents/CSVAgent/CSVAgent.ts b/packages/components/nodes/agents/CSVAgent/CSVAgent.ts index 5d61bcda..70c35cd9 100644 --- a/packages/components/nodes/agents/CSVAgent/CSVAgent.ts +++ b/packages/components/nodes/agents/CSVAgent/CSVAgent.ts @@ -112,6 +112,7 @@ class CSV_Agents implements INode { const chatflowid = options.chatflowid for (const file of files) { + if (!file) continue const fileData = await getFileFromStorage(file, chatflowid) base64String += fileData.toString('base64') } @@ -123,6 +124,7 @@ class CSV_Agents implements INode { } for (const file of files) { + if (!file) continue const splitDataURI = file.split(',') splitDataURI.pop() base64String += splitDataURI.pop() ?? '' diff --git a/packages/components/nodes/chatmodels/AWSBedrock/AWSChatBedrock.ts b/packages/components/nodes/chatmodels/AWSBedrock/AWSChatBedrock.ts index 0c6f5a15..c7b40c7f 100644 --- a/packages/components/nodes/chatmodels/AWSBedrock/AWSChatBedrock.ts +++ b/packages/components/nodes/chatmodels/AWSBedrock/AWSChatBedrock.ts @@ -89,7 +89,7 @@ class AWSChatBedrock_ChatModels implements INode { name: 'allowImageUploads', type: 'boolean', description: - 'Only works with claude-3-* models when image is being uploaded from chat. Compatible with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent', + 'Only works with claude-3-* models when image is being uploaded from chat. Compatible with LLMChain, Conversation Chain, ReAct Agent, Conversational Agent, Tool Agent', default: false, optional: true } diff --git a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts index 97d94764..b2445cc5 100644 --- a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts @@ -106,7 +106,7 @@ class AzureChatOpenAI_ChatModels implements INode { name: 'allowImageUploads', type: 'boolean', description: - 'Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent', + 'Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, Conversational Agent, Tool Agent', default: false, optional: true }, diff --git a/packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts b/packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts index 31ae7af2..5d2451a1 100644 --- a/packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts +++ b/packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts @@ -84,7 +84,7 @@ class ChatAnthropic_ChatModels implements INode { name: 'allowImageUploads', type: 'boolean', description: - 'Automatically uses claude-3-* models when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent', + 'Automatically uses claude-3-* models when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, Conversational Agent, Tool Agent', default: false, optional: true } diff --git a/packages/components/nodes/chatmodels/ChatAnthropic/FlowiseChatAnthropic.ts b/packages/components/nodes/chatmodels/ChatAnthropic/FlowiseChatAnthropic.ts index 9bc52292..40025164 100644 --- a/packages/components/nodes/chatmodels/ChatAnthropic/FlowiseChatAnthropic.ts +++ b/packages/components/nodes/chatmodels/ChatAnthropic/FlowiseChatAnthropic.ts @@ -1,5 +1,5 @@ import { AnthropicInput, ChatAnthropic as LangchainChatAnthropic } from '@langchain/anthropic' -import { BaseLLMParams } from '@langchain/core/language_models/llms' +import { type BaseChatModelParams } from '@langchain/core/language_models/chat_models' import { IVisionChatModal, IMultiModalOption } from '../../../src' export class ChatAnthropic extends LangchainChatAnthropic implements IVisionChatModal { @@ -8,8 +8,9 @@ export class ChatAnthropic extends LangchainChatAnthropic implements IVisionChat multiModalOption: IMultiModalOption id: string - constructor(id: string, fields: Partial & BaseLLMParams & { anthropicApiKey?: string }) { - super(fields) + constructor(id: string, fields?: Partial & BaseChatModelParams) { + // @ts-ignore + super(fields ?? {}) this.id = id this.configuredModel = fields?.modelName || '' this.configuredMaxToken = fields?.maxTokens ?? 2048 diff --git a/packages/components/nodes/chatmodels/ChatGoogleGenerativeAI/ChatGoogleGenerativeAI.ts b/packages/components/nodes/chatmodels/ChatGoogleGenerativeAI/ChatGoogleGenerativeAI.ts index 76287229..d510db42 100644 --- a/packages/components/nodes/chatmodels/ChatGoogleGenerativeAI/ChatGoogleGenerativeAI.ts +++ b/packages/components/nodes/chatmodels/ChatGoogleGenerativeAI/ChatGoogleGenerativeAI.ts @@ -145,7 +145,7 @@ class GoogleGenerativeAI_ChatModels implements INode { name: 'allowImageUploads', type: 'boolean', description: - 'Automatically uses vision model when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent', + 'Automatically uses vision model when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, Conversational Agent, Tool Agent', default: false, optional: true } diff --git a/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts b/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts index e8378a80..532b8d5b 100644 --- a/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts @@ -115,7 +115,7 @@ class ChatOpenAI_ChatModels implements INode { name: 'allowImageUploads', type: 'boolean', description: - 'Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent', + 'Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, Conversational Agent, Tool Agent', default: false, optional: true }, diff --git a/packages/components/nodes/documentloaders/Csv/Csv.ts b/packages/components/nodes/documentloaders/Csv/Csv.ts index 749c3e55..d330ef42 100644 --- a/packages/components/nodes/documentloaders/Csv/Csv.ts +++ b/packages/components/nodes/documentloaders/Csv/Csv.ts @@ -108,6 +108,7 @@ class Csv_DocumentLoaders implements INode { const chatflowid = options.chatflowid for (const file of files) { + if (!file) continue const fileData = await getFileFromStorage(file, chatflowid) const blob = new Blob([fileData]) const loader = new CSVLoader(blob, columnName.trim().length === 0 ? undefined : columnName.trim()) @@ -127,6 +128,7 @@ class Csv_DocumentLoaders implements INode { } for (const file of files) { + if (!file) continue const splitDataURI = file.split(',') splitDataURI.pop() const bf = Buffer.from(splitDataURI.pop() || '', 'base64') diff --git a/packages/components/nodes/documentloaders/Docx/Docx.ts b/packages/components/nodes/documentloaders/Docx/Docx.ts index 11dbfffc..97687418 100644 --- a/packages/components/nodes/documentloaders/Docx/Docx.ts +++ b/packages/components/nodes/documentloaders/Docx/Docx.ts @@ -83,6 +83,7 @@ class Docx_DocumentLoaders implements INode { const chatflowid = options.chatflowid for (const file of files) { + if (!file) continue const fileData = await getFileFromStorage(file, chatflowid) const blob = new Blob([fileData]) const loader = new DocxLoader(blob) @@ -103,6 +104,7 @@ class Docx_DocumentLoaders implements INode { } for (const file of files) { + if (!file) continue const splitDataURI = file.split(',') splitDataURI.pop() const bf = Buffer.from(splitDataURI.pop() || '', 'base64') diff --git a/packages/components/nodes/documentloaders/File/File.ts b/packages/components/nodes/documentloaders/File/File.ts new file mode 100644 index 00000000..03697871 --- /dev/null +++ b/packages/components/nodes/documentloaders/File/File.ts @@ -0,0 +1,299 @@ +import { omit } from 'lodash' +import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' +import { TextSplitter } from 'langchain/text_splitter' +import { TextLoader } from 'langchain/document_loaders/fs/text' +import { JSONLinesLoader, JSONLoader } from 'langchain/document_loaders/fs/json' +import { CSVLoader } from '@langchain/community/document_loaders/fs/csv' +import { PDFLoader } from '@langchain/community/document_loaders/fs/pdf' +import { DocxLoader } from '@langchain/community/document_loaders/fs/docx' +import { BaseDocumentLoader } from 'langchain/document_loaders/base' +import { Document } from '@langchain/core/documents' +import { getFileFromStorage } from '../../../src/storageUtils' +import { mapMimeTypeToExt } from '../../../src/utils' + +class File_DocumentLoaders implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + + constructor() { + this.label = 'File Loader' + this.name = 'fileLoader' + this.version = 1.0 + this.type = 'Document' + this.icon = 'file.svg' + this.category = 'Document Loaders' + this.description = `A generic file loader that can load txt, json, csv, docx, pdf, and other files` + this.baseClasses = [this.type] + this.inputs = [ + { + label: 'File', + name: 'file', + type: 'file', + fileType: '*' + }, + { + label: 'Text Splitter', + name: 'textSplitter', + type: 'TextSplitter', + optional: true + }, + { + label: 'Pdf Usage', + name: 'pdfUsage', + type: 'options', + description: 'Only when loading PDF files', + options: [ + { + label: 'One document per page', + name: 'perPage' + }, + { + label: 'One document per file', + name: 'perFile' + } + ], + default: 'perPage', + optional: true, + additionalParams: true + }, + { + label: 'JSONL Pointer Extraction', + name: 'pointerName', + type: 'string', + description: 'Only when loading JSONL files', + placeholder: '', + optional: true, + additionalParams: true + }, + { + label: 'Additional Metadata', + name: 'metadata', + type: 'json', + description: 'Additional metadata to be added to the extracted documents', + optional: true, + additionalParams: true + }, + { + label: 'Omit Metadata Keys', + name: 'omitMetadataKeys', + type: 'string', + rows: 4, + description: + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', + placeholder: 'key1, key2, key3.nestedKey1', + optional: true, + additionalParams: true + } + ] + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const textSplitter = nodeData.inputs?.textSplitter as TextSplitter + const fileBase64 = nodeData.inputs?.file as string + const metadata = nodeData.inputs?.metadata + const pdfUsage = nodeData.inputs?.pdfUsage + const pointerName = nodeData.inputs?.pointerName as string + const _omitMetadataKeys = nodeData.inputs?.omitMetadataKeys as string + + let omitMetadataKeys: string[] = [] + if (_omitMetadataKeys) { + omitMetadataKeys = _omitMetadataKeys.split(',').map((key) => key.trim()) + } + + let files: string[] = [] + const fileBlobs: { blob: Blob; ext: string }[] = [] + + //FILE-STORAGE::["CONTRIBUTING.md","LICENSE.md","README.md"] + const totalFiles = getOverrideFileInputs(nodeData) || fileBase64 + if (totalFiles.startsWith('FILE-STORAGE::')) { + const fileName = totalFiles.replace('FILE-STORAGE::', '') + if (fileName.startsWith('[') && fileName.endsWith(']')) { + files = JSON.parse(fileName) + } else { + files = [fileName] + } + const chatflowid = options.chatflowid + + for (const file of files) { + if (!file) continue + const fileData = await getFileFromStorage(file, chatflowid) + const blob = new Blob([fileData]) + fileBlobs.push({ blob, ext: file.split('.').pop() || '' }) + } + } else { + if (totalFiles.startsWith('[') && totalFiles.endsWith(']')) { + files = JSON.parse(totalFiles) + } else { + files = [totalFiles] + } + + for (const file of files) { + if (!file) continue + const splitDataURI = file.split(',') + splitDataURI.pop() + const bf = Buffer.from(splitDataURI.pop() || '', 'base64') + const blob = new Blob([bf]) + + let extension = '' + // eslint-disable-next-line no-useless-escape + const match = file.match(/^data:([A-Za-z-+\/]+);base64,/) + + if (!match) { + fileBlobs.push({ + blob, + ext: extension + }) + } else { + const mimeType = match[1] + fileBlobs.push({ + blob, + ext: mapMimeTypeToExt(mimeType) + }) + } + } + } + + const loader = new MultiFileLoader(fileBlobs, { + json: (blob) => new JSONLoader(blob), + jsonl: (blob) => new JSONLinesLoader(blob, '/' + pointerName.trim()), + txt: (blob) => new TextLoader(blob), + csv: (blob) => new CSVLoader(blob), + xls: (blob) => new CSVLoader(blob), + xlsx: (blob) => new CSVLoader(blob), + docx: (blob) => new DocxLoader(blob), + doc: (blob) => new DocxLoader(blob), + pdf: (blob) => + pdfUsage === 'perFile' + ? // @ts-ignore + new PDFLoader(blob, { splitPages: false, pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') }) + : // @ts-ignore + new PDFLoader(blob, { pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') }), + '': (blob) => new TextLoader(blob) + }) + let docs = [] + + if (textSplitter) { + docs = await loader.load() + docs = await textSplitter.splitDocuments(docs) + } else { + docs = await loader.load() + } + + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + docs = docs.map((doc) => ({ + ...doc, + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) + })) + } else { + docs = docs.map((doc) => ({ + ...doc, + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) + })) + } + + return docs + } +} + +const getOverrideFileInputs = (nodeData: INodeData) => { + const txtFileBase64 = nodeData.inputs?.txtFile as string + const pdfFileBase64 = nodeData.inputs?.pdfFile as string + const jsonFileBase64 = nodeData.inputs?.jsonFile as string + const csvFileBase64 = nodeData.inputs?.csvFile as string + const jsonlinesFileBase64 = nodeData.inputs?.jsonlinesFile as string + const docxFileBase64 = nodeData.inputs?.docxFile as string + const yamlFileBase64 = nodeData.inputs?.yamlFile as string + + const removePrefix = (storageFile: string): string[] => { + const fileName = storageFile.replace('FILE-STORAGE::', '') + if (fileName.startsWith('[') && fileName.endsWith(']')) { + return JSON.parse(fileName) + } + return [fileName] + } + + // If exists, combine all file inputs into an array + const files: string[] = [] + if (txtFileBase64) { + files.push(...removePrefix(txtFileBase64)) + } + if (pdfFileBase64) { + files.push(...removePrefix(pdfFileBase64)) + } + if (jsonFileBase64) { + files.push(...removePrefix(jsonFileBase64)) + } + if (csvFileBase64) { + files.push(...removePrefix(csvFileBase64)) + } + if (jsonlinesFileBase64) { + files.push(...removePrefix(jsonlinesFileBase64)) + } + if (docxFileBase64) { + files.push(...removePrefix(docxFileBase64)) + } + if (yamlFileBase64) { + files.push(...removePrefix(yamlFileBase64)) + } + + return files.length ? `FILE-STORAGE::${JSON.stringify(files)}` : '' +} + +interface LoadersMapping { + [extension: string]: (blob: Blob) => BaseDocumentLoader +} + +class MultiFileLoader extends BaseDocumentLoader { + constructor(public fileBlobs: { blob: Blob; ext: string }[], public loaders: LoadersMapping) { + super() + + if (Object.keys(loaders).length === 0) { + throw new Error('Must provide at least one loader') + } + } + + public async load(): Promise { + const documents: Document[] = [] + + for (const fileBlob of this.fileBlobs) { + const loaderFactory = this.loaders[fileBlob.ext] + if (loaderFactory) { + const loader = loaderFactory(fileBlob.blob) + documents.push(...(await loader.load())) + } else { + throw new Error(`Error loading file`) + } + } + + return documents + } +} + +module.exports = { nodeClass: File_DocumentLoaders } diff --git a/packages/components/nodes/documentloaders/File/file.svg b/packages/components/nodes/documentloaders/File/file.svg new file mode 100644 index 00000000..8963ffbc --- /dev/null +++ b/packages/components/nodes/documentloaders/File/file.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/components/nodes/documentloaders/Folder/Folder.ts b/packages/components/nodes/documentloaders/Folder/Folder.ts index fe289f84..d200a7ac 100644 --- a/packages/components/nodes/documentloaders/Folder/Folder.ts +++ b/packages/components/nodes/documentloaders/Folder/Folder.ts @@ -3,7 +3,7 @@ import { INode, INodeData, INodeParams } from '../../../src/Interface' import { TextSplitter } from 'langchain/text_splitter' import { TextLoader } from 'langchain/document_loaders/fs/text' import { DirectoryLoader } from 'langchain/document_loaders/fs/directory' -import { JSONLoader } from 'langchain/document_loaders/fs/json' +import { JSONLinesLoader, JSONLoader } from 'langchain/document_loaders/fs/json' import { CSVLoader } from '@langchain/community/document_loaders/fs/csv' import { PDFLoader } from '@langchain/community/document_loaders/fs/pdf' import { DocxLoader } from '@langchain/community/document_loaders/fs/docx' @@ -22,7 +22,7 @@ class Folder_DocumentLoaders implements INode { constructor() { this.label = 'Folder with Files' this.name = 'folderFiles' - this.version = 2.0 + this.version = 3.0 this.type = 'Document' this.icon = 'folder.svg' this.category = 'Document Loaders' @@ -51,6 +51,7 @@ class Folder_DocumentLoaders implements INode { label: 'Pdf Usage', name: 'pdfUsage', type: 'options', + description: 'Only when loading PDF files', options: [ { label: 'One document per page', @@ -65,6 +66,15 @@ class Folder_DocumentLoaders implements INode { optional: true, additionalParams: true }, + { + label: 'JSONL Pointer Extraction', + name: 'pointerName', + type: 'string', + description: 'Only when loading JSONL files', + placeholder: '', + optional: true, + additionalParams: true + }, { label: 'Additional Metadata', name: 'metadata', @@ -93,6 +103,7 @@ class Folder_DocumentLoaders implements INode { const metadata = nodeData.inputs?.metadata const recursive = nodeData.inputs?.recursive as boolean const pdfUsage = nodeData.inputs?.pdfUsage + const pointerName = nodeData.inputs?.pointerName as string const _omitMetadataKeys = nodeData.inputs?.omitMetadataKeys as string let omitMetadataKeys: string[] = [] @@ -104,8 +115,12 @@ class Folder_DocumentLoaders implements INode { folderPath, { '.json': (path) => new JSONLoader(path), + '.jsonl': (blob) => new JSONLinesLoader(blob, '/' + pointerName.trim()), '.txt': (path) => new TextLoader(path), '.csv': (path) => new CSVLoader(path), + '.xls': (path) => new CSVLoader(path), + '.xlsx': (path) => new CSVLoader(path), + '.doc': (path) => new DocxLoader(path), '.docx': (path) => new DocxLoader(path), '.pdf': (path) => pdfUsage === 'perFile' diff --git a/packages/components/nodes/documentloaders/Json/Json.ts b/packages/components/nodes/documentloaders/Json/Json.ts index ba017d89..4084b359 100644 --- a/packages/components/nodes/documentloaders/Json/Json.ts +++ b/packages/components/nodes/documentloaders/Json/Json.ts @@ -99,6 +99,7 @@ class Json_DocumentLoaders implements INode { const chatflowid = options.chatflowid for (const file of files) { + if (!file) continue const fileData = await getFileFromStorage(file, chatflowid) const blob = new Blob([fileData]) const loader = new JSONLoader(blob, pointers.length != 0 ? pointers : undefined) @@ -119,6 +120,7 @@ class Json_DocumentLoaders implements INode { } for (const file of files) { + if (!file) continue const splitDataURI = file.split(',') splitDataURI.pop() const bf = Buffer.from(splitDataURI.pop() || '', 'base64') diff --git a/packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts b/packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts index 1a4cbb61..2c895e8f 100644 --- a/packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts +++ b/packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts @@ -93,6 +93,7 @@ class Jsonlines_DocumentLoaders implements INode { const chatflowid = options.chatflowid for (const file of files) { + if (!file) continue const fileData = await getFileFromStorage(file, chatflowid) const blob = new Blob([fileData]) const loader = new JSONLinesLoader(blob, pointer) @@ -113,6 +114,7 @@ class Jsonlines_DocumentLoaders implements INode { } for (const file of files) { + if (!file) continue const splitDataURI = file.split(',') splitDataURI.pop() const bf = Buffer.from(splitDataURI.pop() || '', 'base64') diff --git a/packages/components/nodes/documentloaders/Pdf/Pdf.ts b/packages/components/nodes/documentloaders/Pdf/Pdf.ts index 95566f59..48e94bae 100644 --- a/packages/components/nodes/documentloaders/Pdf/Pdf.ts +++ b/packages/components/nodes/documentloaders/Pdf/Pdf.ts @@ -109,6 +109,7 @@ class Pdf_DocumentLoaders implements INode { const chatflowid = options.chatflowid for (const file of files) { + if (!file) continue const fileData = await getFileFromStorage(file, chatflowid) const bf = Buffer.from(fileData) await this.extractDocs(usage, bf, legacyBuild, textSplitter, docs) @@ -121,6 +122,7 @@ class Pdf_DocumentLoaders implements INode { } for (const file of files) { + if (!file) continue const splitDataURI = file.split(',') splitDataURI.pop() const bf = Buffer.from(splitDataURI.pop() || '', 'base64') diff --git a/packages/components/nodes/documentloaders/Text/Text.ts b/packages/components/nodes/documentloaders/Text/Text.ts index 13aae026..950107d6 100644 --- a/packages/components/nodes/documentloaders/Text/Text.ts +++ b/packages/components/nodes/documentloaders/Text/Text.ts @@ -101,6 +101,7 @@ class Text_DocumentLoaders implements INode { const chatflowid = options.chatflowid for (const file of files) { + if (!file) continue const fileData = await getFileFromStorage(file, chatflowid) const blob = new Blob([fileData]) const loader = new TextLoader(blob) @@ -121,6 +122,7 @@ class Text_DocumentLoaders implements INode { } for (const file of files) { + if (!file) continue const splitDataURI = file.split(',') splitDataURI.pop() const bf = Buffer.from(splitDataURI.pop() || '', 'base64') diff --git a/packages/components/nodes/documentloaders/Unstructured/UnstructuredFile.ts b/packages/components/nodes/documentloaders/Unstructured/UnstructuredFile.ts index df3bb09f..beb5e772 100644 --- a/packages/components/nodes/documentloaders/Unstructured/UnstructuredFile.ts +++ b/packages/components/nodes/documentloaders/Unstructured/UnstructuredFile.ts @@ -496,6 +496,7 @@ class UnstructuredFile_DocumentLoaders implements INode { const chatflowid = options.chatflowid for (const file of files) { + if (!file) continue const fileData = await getFileFromStorage(file, chatflowid) const loaderDocs = await loader.loadAndSplitBuffer(fileData, file) docs.push(...loaderDocs) @@ -508,6 +509,7 @@ class UnstructuredFile_DocumentLoaders implements INode { } for (const file of files) { + if (!file) continue const splitDataURI = file.split(',') const filename = splitDataURI.pop()?.split(':')[1] ?? '' const bf = Buffer.from(splitDataURI.pop() || '', 'base64') diff --git a/packages/components/nodes/multiagents/Supervisor/Supervisor.ts b/packages/components/nodes/multiagents/Supervisor/Supervisor.ts index cccd2348..1cd78eae 100644 --- a/packages/components/nodes/multiagents/Supervisor/Supervisor.ts +++ b/packages/components/nodes/multiagents/Supervisor/Supervisor.ts @@ -209,11 +209,11 @@ class Supervisor_MultiAgents implements INode { prompt = messages.prompt multiModalMessageContent = messages.multiModalMessageContent - if (llm.bindTools === undefined) { + if ((llm as any).bindTools === undefined) { throw new Error(`This agent only compatible with function calling models.`) } - const modelWithTool = llm.bindTools([tool]) + const modelWithTool = (llm as any).bindTools([tool]) const outputParser = new ToolCallingAgentOutputParser() @@ -464,11 +464,11 @@ class Supervisor_MultiAgents implements INode { prompt = messages.prompt multiModalMessageContent = messages.multiModalMessageContent - if (llm.bindTools === undefined) { + if ((llm as any).bindTools === undefined) { throw new Error(`This agent only compatible with function calling models.`) } - const modelWithTool = llm.bindTools([tool]) + const modelWithTool = (llm as any).bindTools([tool]) const outputParser = new ToolCallingAgentOutputParser() diff --git a/packages/components/nodes/vectorstores/Milvus/Milvus.ts b/packages/components/nodes/vectorstores/Milvus/Milvus.ts index 74cf4717..469c9df1 100644 --- a/packages/components/nodes/vectorstores/Milvus/Milvus.ts +++ b/packages/components/nodes/vectorstores/Milvus/Milvus.ts @@ -4,7 +4,8 @@ import { Document } from '@langchain/core/documents' import { MilvusLibArgs, Milvus } from '@langchain/community/vectorstores/milvus' import { Embeddings } from '@langchain/core/embeddings' import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' -import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { FLOWISE_CHATID, getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { howToUseFileUpload } from '../VectorStoreUtils' interface InsertRow { [x: string]: string | number[] @@ -27,7 +28,7 @@ class Milvus_VectorStores implements INode { constructor() { this.label = 'Milvus' this.name = 'milvus' - this.version = 1.0 + this.version = 2.0 this.type = 'Milvus' this.icon = 'milvus.svg' this.category = 'Vector Stores' @@ -64,6 +65,18 @@ class Milvus_VectorStores implements INode { name: 'milvusCollection', type: 'string' }, + { + label: 'File Upload', + name: 'fileUpload', + description: 'Allow file upload on the chat', + hint: { + label: 'How to use', + value: howToUseFileUpload + }, + type: 'boolean', + additionalParams: true, + optional: true + }, { label: 'Milvus Text Field', name: 'milvusTextField', @@ -116,6 +129,7 @@ class Milvus_VectorStores implements INode { // embeddings const docs = nodeData.inputs?.document as Document[] const embeddings = nodeData.inputs?.embeddings as Embeddings + const isFileUploadEnabled = nodeData.inputs?.fileUpload as boolean // credential const credentialData = await getCredentialData(nodeData.credential ?? '', options) @@ -135,6 +149,9 @@ class Milvus_VectorStores implements INode { const finalDocs = [] for (let i = 0; i < flattenDocs.length; i += 1) { if (flattenDocs[i] && flattenDocs[i].pageContent) { + if (isFileUploadEnabled && options.chatId) { + flattenDocs[i].metadata = { ...flattenDocs[i].metadata, [FLOWISE_CHATID]: options.chatId } + } finalDocs.push(new Document(flattenDocs[i])) } } @@ -158,8 +175,9 @@ class Milvus_VectorStores implements INode { // server setup const address = nodeData.inputs?.milvusServerUrl as string const collectionName = nodeData.inputs?.milvusCollection as string - const milvusFilter = nodeData.inputs?.milvusFilter as string + const _milvusFilter = nodeData.inputs?.milvusFilter as string const textField = nodeData.inputs?.milvusTextField as string + const isFileUploadEnabled = nodeData.inputs?.fileUpload as boolean // embeddings const embeddings = nodeData.inputs?.embeddings as Embeddings @@ -186,6 +204,12 @@ class Milvus_VectorStores implements INode { if (milvusUser) milVusArgs.username = milvusUser if (milvusPassword) milVusArgs.password = milvusPassword + let milvusFilter = _milvusFilter + if (isFileUploadEnabled && options.chatId) { + if (milvusFilter) milvusFilter += ` OR ${FLOWISE_CHATID} == "${options.chatId}" OR NOT EXISTS(${FLOWISE_CHATID})` + else milvusFilter = `${FLOWISE_CHATID} == "${options.chatId}" OR NOT EXISTS(${FLOWISE_CHATID})` + } + const vectorStore = await Milvus.fromExistingCollection(embeddings, milVusArgs) // Avoid Illegal Invocation diff --git a/packages/components/nodes/vectorstores/Pinecone/Pinecone.ts b/packages/components/nodes/vectorstores/Pinecone/Pinecone.ts index b7af388c..19ff9286 100644 --- a/packages/components/nodes/vectorstores/Pinecone/Pinecone.ts +++ b/packages/components/nodes/vectorstores/Pinecone/Pinecone.ts @@ -5,8 +5,8 @@ import { Embeddings } from '@langchain/core/embeddings' import { Document } from '@langchain/core/documents' import { VectorStore } from '@langchain/core/vectorstores' import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' -import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' -import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils' +import { FLOWISE_CHATID, getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { addMMRInputParams, howToUseFileUpload, resolveVectorStoreOrRetriever } from '../VectorStoreUtils' import { index } from '../../../src/indexing' let pineconeClientSingleton: Pinecone @@ -43,7 +43,7 @@ class Pinecone_VectorStores implements INode { constructor() { this.label = 'Pinecone' this.name = 'pinecone' - this.version = 4.0 + this.version = 5.0 this.type = 'Pinecone' this.icon = 'pinecone.svg' this.category = 'Vector Stores' @@ -88,6 +88,18 @@ class Pinecone_VectorStores implements INode { additionalParams: true, optional: true }, + { + label: 'File Upload', + name: 'fileUpload', + description: 'Allow file upload on the chat', + hint: { + label: 'How to use', + value: howToUseFileUpload + }, + type: 'boolean', + additionalParams: true, + optional: true + }, { label: 'Pinecone Text Key', name: 'pineconeTextKey', @@ -138,6 +150,7 @@ class Pinecone_VectorStores implements INode { const embeddings = nodeData.inputs?.embeddings as Embeddings const recordManager = nodeData.inputs?.recordManager const pineconeTextKey = nodeData.inputs?.pineconeTextKey as string + const isFileUploadEnabled = nodeData.inputs?.fileUpload as boolean const credentialData = await getCredentialData(nodeData.credential ?? '', options) const pineconeApiKey = getCredentialParam('pineconeApiKey', credentialData, nodeData) @@ -150,6 +163,9 @@ class Pinecone_VectorStores implements INode { const finalDocs = [] for (let i = 0; i < flattenDocs.length; i += 1) { if (flattenDocs[i] && flattenDocs[i].pageContent) { + if (isFileUploadEnabled && options.chatId) { + flattenDocs[i].metadata = { ...flattenDocs[i].metadata, [FLOWISE_CHATID]: options.chatId } + } finalDocs.push(new Document(flattenDocs[i])) } } @@ -232,6 +248,7 @@ class Pinecone_VectorStores implements INode { const pineconeMetadataFilter = nodeData.inputs?.pineconeMetadataFilter const embeddings = nodeData.inputs?.embeddings as Embeddings const pineconeTextKey = nodeData.inputs?.pineconeTextKey as string + const isFileUploadEnabled = nodeData.inputs?.fileUpload as boolean const credentialData = await getCredentialData(nodeData.credential ?? '', options) const pineconeApiKey = getCredentialParam('pineconeApiKey', credentialData, nodeData) @@ -250,6 +267,14 @@ class Pinecone_VectorStores implements INode { const metadatafilter = typeof pineconeMetadataFilter === 'object' ? pineconeMetadataFilter : JSON.parse(pineconeMetadataFilter) obj.filter = metadatafilter } + if (isFileUploadEnabled && options.chatId) { + obj.filter = obj.filter || {} + obj.filter.$or = [ + ...(obj.filter.$or || []), + { [FLOWISE_CHATID]: { $eq: options.chatId } }, + { [FLOWISE_CHATID]: { $exists: false } } + ] + } const vectorStore = (await PineconeStore.fromExistingIndex(embeddings, obj)) as unknown as VectorStore diff --git a/packages/components/nodes/vectorstores/Postgres/Postgres.ts b/packages/components/nodes/vectorstores/Postgres/Postgres.ts index 4a4d5443..b5827dc3 100644 --- a/packages/components/nodes/vectorstores/Postgres/Postgres.ts +++ b/packages/components/nodes/vectorstores/Postgres/Postgres.ts @@ -5,8 +5,9 @@ import { Embeddings } from '@langchain/core/embeddings' import { Document } from '@langchain/core/documents' import { TypeORMVectorStore, TypeORMVectorStoreDocument } from '@langchain/community/vectorstores/typeorm' import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' -import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { FLOWISE_CHATID, getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { index } from '../../../src/indexing' +import { howToUseFileUpload } from '../VectorStoreUtils' class Postgres_VectorStores implements INode { label: string @@ -25,7 +26,7 @@ class Postgres_VectorStores implements INode { constructor() { this.label = 'Postgres' this.name = 'postgres' - this.version = 5.0 + this.version = 6.0 this.type = 'Postgres' this.icon = 'postgres.svg' this.category = 'Vector Stores' @@ -82,6 +83,18 @@ class Postgres_VectorStores implements INode { additionalParams: true, optional: true }, + { + label: 'File Upload', + name: 'fileUpload', + description: 'Allow file upload on the chat', + hint: { + label: 'How to use', + value: howToUseFileUpload + }, + type: 'boolean', + additionalParams: true, + optional: true + }, { label: 'Additional Configuration', name: 'additionalConfig', @@ -132,6 +145,7 @@ class Postgres_VectorStores implements INode { const embeddings = nodeData.inputs?.embeddings as Embeddings const additionalConfig = nodeData.inputs?.additionalConfig as string const recordManager = nodeData.inputs?.recordManager + const isFileUploadEnabled = nodeData.inputs?.fileUpload as boolean let additionalConfiguration = {} if (additionalConfig) { @@ -161,6 +175,9 @@ class Postgres_VectorStores implements INode { const finalDocs = [] for (let i = 0; i < flattenDocs.length; i += 1) { if (flattenDocs[i] && flattenDocs[i].pageContent) { + if (isFileUploadEnabled && options.chatId) { + flattenDocs[i].metadata = { ...flattenDocs[i].metadata, [FLOWISE_CHATID]: options.chatId } + } finalDocs.push(new Document(flattenDocs[i])) } } @@ -268,11 +285,20 @@ class Postgres_VectorStores implements INode { const topK = nodeData.inputs?.topK as string const k = topK ? parseFloat(topK) : 4 const _pgMetadataFilter = nodeData.inputs?.pgMetadataFilter + const isFileUploadEnabled = nodeData.inputs?.fileUpload as boolean let pgMetadataFilter: any if (_pgMetadataFilter) { pgMetadataFilter = typeof _pgMetadataFilter === 'object' ? _pgMetadataFilter : JSON.parse(_pgMetadataFilter) } + if (isFileUploadEnabled && options.chatId) { + pgMetadataFilter = pgMetadataFilter || {} + pgMetadataFilter = { + ...pgMetadataFilter, + [FLOWISE_CHATID]: options.chatId, + $notexists: FLOWISE_CHATID // special filter to check if the field does not exist + } + } let additionalConfiguration = {} if (additionalConfig) { @@ -334,12 +360,20 @@ const similaritySearchVectorWithScore = async ( ) => { const embeddingString = `[${query.join(',')}]` let _filter = '{}' - if (filter && typeof filter === 'object') _filter = JSON.stringify(filter) + let notExists = '' + if (filter && typeof filter === 'object') { + if (filter.$notexists) { + notExists = `OR NOT (metadata ? '${filter.$notexists}')` + delete filter.$notexists + } + _filter = JSON.stringify(filter) + } const queryString = ` SELECT *, embedding <=> $1 as "_distance" FROM ${tableName} WHERE metadata @> $2 + ${notExists} ORDER BY "_distance" ASC LIMIT $3;` diff --git a/packages/components/nodes/vectorstores/Qdrant/Qdrant.ts b/packages/components/nodes/vectorstores/Qdrant/Qdrant.ts index f53e3bae..2bdbcf35 100644 --- a/packages/components/nodes/vectorstores/Qdrant/Qdrant.ts +++ b/packages/components/nodes/vectorstores/Qdrant/Qdrant.ts @@ -6,8 +6,9 @@ import { Document } from '@langchain/core/documents' import { QdrantVectorStore, QdrantLibArgs } from '@langchain/qdrant' import { Embeddings } from '@langchain/core/embeddings' import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' -import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { FLOWISE_CHATID, getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { index } from '../../../src/indexing' +import { howToUseFileUpload } from '../VectorStoreUtils' type RetrieverConfig = Partial> type QdrantAddDocumentOptions = { @@ -32,7 +33,7 @@ class Qdrant_VectorStores implements INode { constructor() { this.label = 'Qdrant' this.name = 'qdrant' - this.version = 4.0 + this.version = 5.0 this.type = 'Qdrant' this.icon = 'qdrant.png' this.category = 'Vector Stores' @@ -78,6 +79,18 @@ class Qdrant_VectorStores implements INode { name: 'qdrantCollection', type: 'string' }, + { + label: 'File Upload', + name: 'fileUpload', + description: 'Allow file upload on the chat', + hint: { + label: 'How to use', + value: howToUseFileUpload + }, + type: 'boolean', + additionalParams: true, + optional: true + }, { label: 'Vector Dimension', name: 'qdrantVectorDimension', @@ -188,6 +201,7 @@ class Qdrant_VectorStores implements INode { const _batchSize = nodeData.inputs?.batchSize const contentPayloadKey = nodeData.inputs?.contentPayloadKey || 'content' const metadataPayloadKey = nodeData.inputs?.metadataPayloadKey || 'metadata' + const isFileUploadEnabled = nodeData.inputs?.fileUpload as boolean const credentialData = await getCredentialData(nodeData.credential ?? '', options) const qdrantApiKey = getCredentialParam('qdrantApiKey', credentialData, nodeData) @@ -204,6 +218,9 @@ class Qdrant_VectorStores implements INode { const finalDocs = [] for (let i = 0; i < flattenDocs.length; i += 1) { if (flattenDocs[i] && flattenDocs[i].pageContent) { + if (isFileUploadEnabled && options.chatId) { + flattenDocs[i].metadata = { ...flattenDocs[i].metadata, [FLOWISE_CHATID]: options.chatId } + } finalDocs.push(new Document(flattenDocs[i])) } } @@ -391,6 +408,7 @@ class Qdrant_VectorStores implements INode { let queryFilter = nodeData.inputs?.qdrantFilter const contentPayloadKey = nodeData.inputs?.contentPayloadKey || 'content' const metadataPayloadKey = nodeData.inputs?.metadataPayloadKey || 'metadata' + const isFileUploadEnabled = nodeData.inputs?.fileUpload as boolean const k = topK ? parseFloat(topK) : 4 @@ -434,6 +452,25 @@ class Qdrant_VectorStores implements INode { if (queryFilter) { retrieverConfig.filter = typeof queryFilter === 'object' ? queryFilter : JSON.parse(queryFilter) } + if (isFileUploadEnabled && options.chatId) { + retrieverConfig.filter = retrieverConfig.filter || {} + + retrieverConfig.filter.should = Array.isArray(retrieverConfig.filter.should) ? retrieverConfig.filter.should : [] + + retrieverConfig.filter.should.push( + { + key: `metadata.${FLOWISE_CHATID}`, + match: { + value: options.chatId + } + }, + { + is_empty: { + key: `metadata.${FLOWISE_CHATID}` + } + } + ) + } const vectorStore = await QdrantVectorStore.fromExistingCollection(embeddings, dbConfig) diff --git a/packages/components/nodes/vectorstores/Upstash/Upstash.ts b/packages/components/nodes/vectorstores/Upstash/Upstash.ts index e41ab849..d126daa8 100644 --- a/packages/components/nodes/vectorstores/Upstash/Upstash.ts +++ b/packages/components/nodes/vectorstores/Upstash/Upstash.ts @@ -1,12 +1,12 @@ import { flatten } from 'lodash' import { IndexingResult, INode, INodeOutputsValue, INodeParams, INodeData, ICommonObject } from '../../../src/Interface' -import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { FLOWISE_CHATID, getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { Embeddings } from '@langchain/core/embeddings' import { Document } from '@langchain/core/documents' import { UpstashVectorStore } from '@langchain/community/vectorstores/upstash' import { Index as UpstashIndex } from '@upstash/vector' import { index } from '../../../src/indexing' -import { resolveVectorStoreOrRetriever } from '../VectorStoreUtils' +import { howToUseFileUpload, resolveVectorStoreOrRetriever } from '../VectorStoreUtils' type UpstashVectorStoreParams = { index: UpstashIndex @@ -29,7 +29,7 @@ class Upstash_VectorStores implements INode { constructor() { this.label = 'Upstash Vector' this.name = 'upstash' - this.version = 1.0 + this.version = 2.0 this.type = 'Upstash' this.icon = 'upstash.svg' this.category = 'Vector Stores' @@ -63,6 +63,18 @@ class Upstash_VectorStores implements INode { description: 'Keep track of the record to prevent duplication', optional: true }, + { + label: 'File Upload', + name: 'fileUpload', + description: 'Allow file upload on the chat', + hint: { + label: 'How to use', + value: howToUseFileUpload + }, + type: 'boolean', + additionalParams: true, + optional: true + }, { label: 'Upstash Metadata Filter', name: 'upstashMetadataFilter', @@ -100,6 +112,7 @@ class Upstash_VectorStores implements INode { const docs = nodeData.inputs?.document as Document[] const embeddings = nodeData.inputs?.embeddings as Embeddings const recordManager = nodeData.inputs?.recordManager + const isFileUploadEnabled = nodeData.inputs?.fileUpload as boolean const credentialData = await getCredentialData(nodeData.credential ?? '', options) const UPSTASH_VECTOR_REST_URL = getCredentialParam('UPSTASH_VECTOR_REST_URL', credentialData, nodeData) @@ -114,6 +127,9 @@ class Upstash_VectorStores implements INode { const finalDocs = [] for (let i = 0; i < flattenDocs.length; i += 1) { if (flattenDocs[i] && flattenDocs[i].pageContent) { + if (isFileUploadEnabled && options.chatId) { + flattenDocs[i].metadata = { ...flattenDocs[i].metadata, [FLOWISE_CHATID]: options.chatId } + } finalDocs.push(new Document(flattenDocs[i])) } } @@ -186,6 +202,7 @@ class Upstash_VectorStores implements INode { async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { const upstashMetadataFilter = nodeData.inputs?.upstashMetadataFilter const embeddings = nodeData.inputs?.embeddings as Embeddings + const isFileUploadEnabled = nodeData.inputs?.fileUpload as boolean const credentialData = await getCredentialData(nodeData.credential ?? '', options) const UPSTASH_VECTOR_REST_URL = getCredentialParam('UPSTASH_VECTOR_REST_URL', credentialData, nodeData) @@ -203,6 +220,10 @@ class Upstash_VectorStores implements INode { if (upstashMetadataFilter) { obj.filter = upstashMetadataFilter } + if (isFileUploadEnabled && options.chatId) { + if (upstashMetadataFilter) obj.filter += ` OR ${FLOWISE_CHATID} = "${options.chatId}" OR HAS NOT FIELD ${FLOWISE_CHATID}` + else obj.filter = `${FLOWISE_CHATID} = "${options.chatId}" OR HAS NOT FIELD ${FLOWISE_CHATID}` + } const vectorStore = await UpstashVectorStore.fromExistingIndex(embeddings, obj) diff --git a/packages/components/nodes/vectorstores/Vectara/Vectara.ts b/packages/components/nodes/vectorstores/Vectara/Vectara.ts index 462eb2bd..d7260d10 100644 --- a/packages/components/nodes/vectorstores/Vectara/Vectara.ts +++ b/packages/components/nodes/vectorstores/Vectara/Vectara.ts @@ -194,6 +194,7 @@ class Vectara_VectorStores implements INode { const chatflowid = options.chatflowid for (const file of files) { + if (!file) continue const fileData = await getFileFromStorage(file, chatflowid) const blob = new Blob([fileData]) vectaraFiles.push({ blob: blob, fileName: getFileName(file) }) @@ -206,6 +207,7 @@ class Vectara_VectorStores implements INode { } for (const file of files) { + if (!file) continue const splitDataURI = file.split(',') splitDataURI.pop() const bf = Buffer.from(splitDataURI.pop() || '', 'base64') diff --git a/packages/components/nodes/vectorstores/VectorStoreUtils.ts b/packages/components/nodes/vectorstores/VectorStoreUtils.ts index 8b75337c..c5bc941b 100644 --- a/packages/components/nodes/vectorstores/VectorStoreUtils.ts +++ b/packages/components/nodes/vectorstores/VectorStoreUtils.ts @@ -83,3 +83,19 @@ export const addMMRInputParams = (inputs: any[]) => { inputs.push(...mmrInputParams) } + +export const howToUseFileUpload = ` +**File Upload** + +This allows file upload on the chat. Uploaded files will be upserted on the fly to the vector store. + +**Note:** +- You can only turn on file upload for one vector store at a time. +- At least one Document Loader node should be connected to the document input. +- Document Loader should be file types like PDF, DOCX, TXT, etc. + +**How it works** +- Uploaded files will have the metadata updated with the chatId. +- This will allow the file to be associated with the chatId. +- When querying, metadata will be filtered by chatId to retrieve files associated with the chatId. +` diff --git a/packages/components/src/utils.ts b/packages/components/src/utils.ts index 2b5d68a2..317103c8 100644 --- a/packages/components/src/utils.ts +++ b/packages/components/src/utils.ts @@ -11,6 +11,8 @@ import { AIMessage, HumanMessage, BaseMessage } from '@langchain/core/messages' export const numberOrExpressionRegex = '^(\\d+\\.?\\d*|{{.*}})$' //return true if string consists only numbers OR expression {{}} export const notEmptyRegex = '(.|\\s)*\\S(.|\\s)*' //return true if string is not empty or blank +export const FLOWISE_CHATID = 'flowise_chatId' + /* * List of dependencies allowed to be import in vm2 */ @@ -815,3 +817,67 @@ export const getVersion: () => Promise<{ version: string }> = async () => { throw new Error('None of the package.json paths could be parsed') } + +/** + * Map MimeType to InputField + * @param {string} mimeType + * @returns {string} + */ +export const mapMimeTypeToInputField = (mimeType: string) => { + switch (mimeType) { + case 'text/plain': + return 'txtFile' + case 'application/pdf': + return 'pdfFile' + case 'application/json': + return 'jsonFile' + case 'text/csv': + return 'csvFile' + case 'application/json-lines': + case 'application/jsonl': + case 'text/jsonl': + return 'jsonlinesFile' + case 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': + return 'docxFile' + case 'application/vnd.yaml': + case 'application/x-yaml': + case 'text/vnd.yaml': + case 'text/x-yaml': + case 'text/yaml': + return 'yamlFile' + default: + return 'txtFile' + } +} + +/** + * Map MimeType to Extension + * @param {string} mimeType + * @returns {string} + */ +export const mapMimeTypeToExt = (mimeType: string) => { + switch (mimeType) { + case 'text/plain': + return 'txt' + case 'application/pdf': + return 'pdf' + case 'application/json': + return 'json' + case 'text/csv': + return 'csv' + case 'application/json-lines': + case 'application/jsonl': + case 'text/jsonl': + return 'jsonl' + case 'application/msword': + return 'doc' + case 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': + return 'docx' + case 'application/vnd.ms-excel': + return 'xls' + case 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': + return 'xlsx' + default: + return '' + } +} diff --git a/packages/server/marketplaces/chatflows/SQL Prompt.json b/packages/server/marketplaces/chatflows/SQL Prompt.json index 086a348f..4e2524b4 100644 --- a/packages/server/marketplaces/chatflows/SQL Prompt.json +++ b/packages/server/marketplaces/chatflows/SQL Prompt.json @@ -5,7 +5,7 @@ "nodes": [ { "width": 300, - "height": 513, + "height": 511, "id": "promptTemplate_0", "position": { "x": 384.4880563109088, @@ -65,7 +65,7 @@ }, { "width": 300, - "height": 508, + "height": 507, "id": "llmChain_0", "position": { "x": 770.4559230968546, @@ -164,7 +164,7 @@ }, { "width": 300, - "height": 670, + "height": 669, "id": "chatOpenAI_0", "position": { "x": 376.92707114970364, @@ -343,7 +343,7 @@ }, { "width": 300, - "height": 670, + "height": 669, "id": "chatOpenAI_1", "position": { "x": 2653.726672579251, @@ -522,7 +522,7 @@ }, { "width": 300, - "height": 508, + "height": 507, "id": "llmChain_1", "position": { "x": 3089.9937691022837, @@ -621,7 +621,7 @@ }, { "width": 300, - "height": 670, + "height": 674, "id": "customFunction_2", "position": { "x": -19.95227863012829, @@ -709,7 +709,7 @@ }, { "width": 300, - "height": 670, + "height": 674, "id": "customFunction_1", "position": { "x": 1887.4670208331604, @@ -797,7 +797,7 @@ }, { "width": 300, - "height": 513, + "height": 511, "id": "promptTemplate_1", "position": { "x": 2655.2632506040304, @@ -857,7 +857,7 @@ }, { "width": 300, - "height": 305, + "height": 304, "id": "getVariable_1", "position": { "x": 2272.8555266616872, @@ -919,7 +919,7 @@ }, { "width": 300, - "height": 356, + "height": 355, "id": "setVariable_1", "position": { "x": 1516.338224315744, @@ -991,7 +991,7 @@ }, { "width": 300, - "height": 757, + "height": 765, "id": "ifElseFunction_0", "position": { "x": 1147.8020838770517, @@ -1050,7 +1050,7 @@ "inputs": { "functionInputVariables": "{\"sqlQuery\":\"{{llmChain_0.data.instance}}\"}", "functionName": "IF SQL Query contains SELECT and WHERE", - "ifFunction": "const sqlQuery = $sqlQuery.trim();\n\nif (sqlQuery.includes(\"SELECT\") && sqlQuery.includes(\"WHERE\")) {\n return sqlQuery;\n}", + "ifFunction": "const sqlQuery = $sqlQuery.trim();\n\nconst regex = /SELECT\\s.*?(?:\\n|$)/gi;\n\n// Extracting the SQL part\nconst matches = sqlQuery.match(regex);\nconst cleanSql = matches ? matches[0].trim() : \"\";\n\nif (cleanSql.includes(\"SELECT\") && cleanSql.includes(\"WHERE\")) {\n return cleanSql;\n}", "elseFunction": "return $sqlQuery;" }, "outputAnchors": [ @@ -1092,7 +1092,7 @@ }, { "width": 300, - "height": 513, + "height": 511, "id": "promptTemplate_2", "position": { "x": 1193.7489579044463, @@ -1152,7 +1152,7 @@ }, { "width": 300, - "height": 670, + "height": 669, "id": "chatOpenAI_2", "position": { "x": 1545.1023725538003, @@ -1331,7 +1331,7 @@ }, { "width": 300, - "height": 508, + "height": 507, "id": "llmChain_2", "position": { "x": 1914.509823868027, diff --git a/packages/server/src/ChatflowPool.ts b/packages/server/src/ChatflowPool.ts index 325fac56..5f1e3a84 100644 --- a/packages/server/src/ChatflowPool.ts +++ b/packages/server/src/ChatflowPool.ts @@ -16,13 +16,21 @@ export class ChatflowPool { * @param {IReactFlowNode[]} startingNodes * @param {ICommonObject} overrideConfig */ - add(chatflowid: string, endingNodeData: INodeData | undefined, startingNodes: IReactFlowNode[], overrideConfig?: ICommonObject) { + add( + chatflowid: string, + endingNodeData: INodeData | undefined, + startingNodes: IReactFlowNode[], + overrideConfig?: ICommonObject, + chatId?: string + ) { this.activeChatflows[chatflowid] = { startingNodes, endingNodeData, inSync: true } if (overrideConfig) this.activeChatflows[chatflowid].overrideConfig = overrideConfig + if (chatId) this.activeChatflows[chatflowid].chatId = chatId + logger.info(`[server]: Chatflow ${chatflowid} added into ChatflowPool`) } diff --git a/packages/server/src/Interface.ts b/packages/server/src/Interface.ts index 803d044d..f4104576 100644 --- a/packages/server/src/Interface.ts +++ b/packages/server/src/Interface.ts @@ -231,6 +231,7 @@ export interface IActiveChatflows { endingNodeData?: INodeData inSync: boolean overrideConfig?: ICommonObject + chatId?: string } } diff --git a/packages/server/src/routes/vectors/index.ts b/packages/server/src/routes/vectors/index.ts index b0da82b6..cc257bca 100644 --- a/packages/server/src/routes/vectors/index.ts +++ b/packages/server/src/routes/vectors/index.ts @@ -14,6 +14,6 @@ router.post( vectorsController.getRateLimiterMiddleware, vectorsController.upsertVectorMiddleware ) -router.post(['/internal-upsert/', '/internal-upsert/:id'], vectorsController.createInternalUpsert) +router.post(['/internal-upsert/', '/internal-upsert/:id'], upload.array('files'), vectorsController.createInternalUpsert) export default router diff --git a/packages/server/src/utils/buildChatflow.ts b/packages/server/src/utils/buildChatflow.ts index 59a331e1..93d4fd5e 100644 --- a/packages/server/src/utils/buildChatflow.ts +++ b/packages/server/src/utils/buildChatflow.ts @@ -1,5 +1,12 @@ import { Request } from 'express' -import { IFileUpload, convertSpeechToText, ICommonObject, addSingleFileToStorage, addArrayFilesToStorage } from 'flowise-components' +import { + IFileUpload, + convertSpeechToText, + ICommonObject, + addSingleFileToStorage, + addArrayFilesToStorage, + mapMimeTypeToInputField +} from 'flowise-components' import { StatusCodes } from 'http-status-codes' import { IncomingInput, @@ -18,7 +25,6 @@ import { ChatFlow } from '../database/entities/ChatFlow' import { Server } from 'socket.io' import { getRunningExpressApp } from '../utils/getRunningExpressApp' import { - mapMimeTypeToInputField, isFlowValidForStream, buildFlow, getTelemetryFlowObj, @@ -32,7 +38,8 @@ import { getMemorySessionId, isSameOverrideConfig, getEndingNodes, - constructGraphs + constructGraphs, + isSameChatId } from '../utils' import { validateChatflowAPIKey } from './validateKey' import { databaseEntities } from '.' @@ -201,6 +208,7 @@ export const utilBuildChatflow = async (req: Request, socketIO?: Server, isInter * - Node Data already exists in pool * - Still in sync (i.e the flow has not been modified since) * - Existing overrideConfig and new overrideConfig are the same + * - Existing chatId and new chatId is the same * - Flow doesn't start with/contain nodes that depend on incomingInput.question ***/ const isFlowReusable = () => { @@ -209,6 +217,7 @@ export const utilBuildChatflow = async (req: Request, socketIO?: Server, isInter Object.prototype.hasOwnProperty.call(appServer.chatflowPool.activeChatflows, chatflowid) && appServer.chatflowPool.activeChatflows[chatflowid].inSync && appServer.chatflowPool.activeChatflows[chatflowid].endingNodeData && + isSameChatId(appServer.chatflowPool.activeChatflows[chatflowid].chatId, chatId) && isSameOverrideConfig( isInternal, appServer.chatflowPool.activeChatflows[chatflowid].overrideConfig, @@ -338,7 +347,7 @@ export const utilBuildChatflow = async (req: Request, socketIO?: Server, isInter ) nodeToExecuteData = reactFlowNodeData - appServer.chatflowPool.add(chatflowid, nodeToExecuteData, startingNodes, incomingInput?.overrideConfig) + appServer.chatflowPool.add(chatflowid, nodeToExecuteData, startingNodes, incomingInput?.overrideConfig, chatId) } logger.debug(`[server]: Running ${nodeToExecuteData.label} (${nodeToExecuteData.id})`) diff --git a/packages/server/src/utils/getUploadsConfig.ts b/packages/server/src/utils/getUploadsConfig.ts index 13670c99..4688d680 100644 --- a/packages/server/src/utils/getUploadsConfig.ts +++ b/packages/server/src/utils/getUploadsConfig.ts @@ -2,14 +2,22 @@ import { StatusCodes } from 'http-status-codes' import { INodeParams } from 'flowise-components' import { ChatFlow } from '../database/entities/ChatFlow' import { getRunningExpressApp } from '../utils/getRunningExpressApp' -import { IUploadFileSizeAndTypes, IReactFlowNode } from '../Interface' +import { IUploadFileSizeAndTypes, IReactFlowNode, IReactFlowEdge } from '../Interface' import { InternalFlowiseError } from '../errors/internalFlowiseError' +type IUploadConfig = { + isSpeechToTextEnabled: boolean + isImageUploadAllowed: boolean + isFileUploadAllowed: boolean + imgUploadSizeAndTypes: IUploadFileSizeAndTypes[] + fileUploadSizeAndTypes: IUploadFileSizeAndTypes[] +} + /** * Method that checks if uploads are enabled in the chatflow * @param {string} chatflowid */ -export const utilGetUploadsConfig = async (chatflowid: string): Promise => { +export const utilGetUploadsConfig = async (chatflowid: string): Promise => { const appServer = getRunningExpressApp() const chatflow = await appServer.AppDataSource.getRepository(ChatFlow).findOneBy({ id: chatflowid @@ -18,21 +26,17 @@ export const utilGetUploadsConfig = async (chatflowid: string): Promise => throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Chatflow ${chatflowid} not found`) } - const uploadAllowedNodes = [ - 'llmChain', - 'conversationChain', - 'reactAgentChat', - 'conversationalAgent', - 'toolAgent', - 'supervisor', - 'seqStart' - ] - const uploadProcessingNodes = ['chatOpenAI', 'chatAnthropic', 'awsChatBedrock', 'azureChatOpenAI', 'chatGoogleGenerativeAI'] - const flowObj = JSON.parse(chatflow.flowData) - const imgUploadSizeAndTypes: IUploadFileSizeAndTypes[] = [] + const nodes: IReactFlowNode[] = flowObj.nodes + const edges: IReactFlowEdge[] = flowObj.edges let isSpeechToTextEnabled = false + let isImageUploadAllowed = false + let isFileUploadAllowed = false + + /* + * Check for STT + */ if (chatflow.speechToText) { const speechToTextProviders = JSON.parse(chatflow.speechToText) for (const provider in speechToTextProviders) { @@ -46,39 +50,72 @@ export const utilGetUploadsConfig = async (chatflowid: string): Promise => } } - let isImageUploadAllowed = false - const nodes: IReactFlowNode[] = flowObj.nodes + /* + * Condition for isFileUploadAllowed + * 1.) vector store with fileUpload = true && connected to a document loader with fileType + */ + const fileUploadSizeAndTypes: IUploadFileSizeAndTypes[] = [] + for (const node of nodes) { + if (node.data.category === 'Vector Stores' && node.data.inputs?.fileUpload) { + // Get the connected document loader node fileTypes + const sourceDocumentEdges = edges.filter( + (edge) => edge.target === node.id && edge.targetHandle === `${node.id}-input-document-Document` + ) + for (const edge of sourceDocumentEdges) { + const sourceNode = nodes.find((node) => node.id === edge.source) + if (!sourceNode) continue + const fileType = sourceNode.data.inputParams.find((param) => param.type === 'file' && param.fileType)?.fileType + if (fileType) { + fileUploadSizeAndTypes.push({ + fileTypes: fileType.split(', '), + maxUploadSize: 500 + }) + isFileUploadAllowed = true + } + } + break + } + } /* * Condition for isImageUploadAllowed - * 1.) one of the uploadAllowedNodes exists - * 2.) one of the uploadProcessingNodes exists + allowImageUploads is ON + * 1.) one of the imgUploadAllowedNodes exists + * 2.) one of the imgUploadLLMNodes exists + allowImageUploads is ON */ - if (!nodes.some((node) => uploadAllowedNodes.includes(node.data.name))) { - return { - isSpeechToTextEnabled, - isImageUploadAllowed: false, - imgUploadSizeAndTypes - } + const imgUploadSizeAndTypes: IUploadFileSizeAndTypes[] = [] + const imgUploadAllowedNodes = [ + 'llmChain', + 'conversationChain', + 'reactAgentChat', + 'conversationalAgent', + 'toolAgent', + 'supervisor', + 'seqStart' + ] + const imgUploadLLMNodes = ['chatOpenAI', 'chatAnthropic', 'awsChatBedrock', 'azureChatOpenAI', 'chatGoogleGenerativeAI'] + + if (nodes.some((node) => imgUploadAllowedNodes.includes(node.data.name))) { + nodes.forEach((node: IReactFlowNode) => { + if (imgUploadLLMNodes.indexOf(node.data.name) > -1) { + // TODO: for now the maxUploadSize is hardcoded to 5MB, we need to add it to the node properties + node.data.inputParams.map((param: INodeParams) => { + if (param.name === 'allowImageUploads' && node.data.inputs?.['allowImageUploads']) { + imgUploadSizeAndTypes.push({ + fileTypes: 'image/gif;image/jpeg;image/png;image/webp;'.split(';'), + maxUploadSize: 5 + }) + isImageUploadAllowed = true + } + }) + } + }) } - nodes.forEach((node: IReactFlowNode) => { - if (uploadProcessingNodes.indexOf(node.data.name) > -1) { - // TODO: for now the maxUploadSize is hardcoded to 5MB, we need to add it to the node properties - node.data.inputParams.map((param: INodeParams) => { - if (param.name === 'allowImageUploads' && node.data.inputs?.['allowImageUploads']) { - imgUploadSizeAndTypes.push({ - fileTypes: 'image/gif;image/jpeg;image/png;image/webp;'.split(';'), - maxUploadSize: 5 - }) - isImageUploadAllowed = true - } - }) - } - }) return { isSpeechToTextEnabled, isImageUploadAllowed, - imgUploadSizeAndTypes + isFileUploadAllowed, + imgUploadSizeAndTypes, + fileUploadSizeAndTypes } } diff --git a/packages/server/src/utils/index.ts b/packages/server/src/utils/index.ts index 50c43f99..2f884b11 100644 --- a/packages/server/src/utils/index.ts +++ b/packages/server/src/utils/index.ts @@ -1074,35 +1074,16 @@ export const isSameOverrideConfig = ( } /** - * Map MimeType to InputField - * @param {string} mimeType - * @returns {Promise} + * @param {string} existingChatId + * @param {string} newChatId + * @returns {boolean} */ -export const mapMimeTypeToInputField = (mimeType: string) => { - switch (mimeType) { - case 'text/plain': - return 'txtFile' - case 'application/pdf': - return 'pdfFile' - case 'application/json': - return 'jsonFile' - case 'text/csv': - return 'csvFile' - case 'application/json-lines': - case 'application/jsonl': - case 'text/jsonl': - return 'jsonlinesFile' - case 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': - return 'docxFile' - case 'application/vnd.yaml': - case 'application/x-yaml': - case 'text/vnd.yaml': - case 'text/x-yaml': - case 'text/yaml': - return 'yamlFile' - default: - return 'txtFile' +export const isSameChatId = (existingChatId?: string, newChatId?: string): boolean => { + if (isEqual(existingChatId, newChatId)) { + return true } + if (!existingChatId && !newChatId) return true + return false } /** diff --git a/packages/server/src/utils/upsertVector.ts b/packages/server/src/utils/upsertVector.ts index d8db1446..a6f67e9b 100644 --- a/packages/server/src/utils/upsertVector.ts +++ b/packages/server/src/utils/upsertVector.ts @@ -1,14 +1,13 @@ import { Request } from 'express' import * as fs from 'fs' import { cloneDeep, omit } from 'lodash' -import { ICommonObject, IMessage, addArrayFilesToStorage } from 'flowise-components' +import { ICommonObject, IMessage, addArrayFilesToStorage, mapMimeTypeToInputField } from 'flowise-components' import telemetryService from '../services/telemetry' import logger from '../utils/logger' import { buildFlow, constructGraphs, getAllConnectedNodes, - mapMimeTypeToInputField, findMemoryNode, getMemorySessionId, getAppVersion, @@ -70,6 +69,9 @@ export const upsertVector = async (req: Request, isInternal: boolean = false) => overrideConfig, stopNodeId: req.body.stopNodeId } + if (req.body.chatId) { + incomingInput.chatId = req.body.chatId + } } /*** Get chatflows and prepare data ***/ @@ -87,10 +89,15 @@ export const upsertVector = async (req: Request, isInternal: boolean = false) => const memoryNode = findMemoryNode(nodes, edges) let sessionId = getMemorySessionId(memoryNode, incomingInput, chatId, isInternal) - const vsNodes = nodes.filter( - (node) => - node.data.category === 'Vector Stores' && !node.data.label.includes('Upsert') && !node.data.label.includes('Load Existing') - ) + const vsNodes = nodes.filter((node) => node.data.category === 'Vector Stores') + + // Get StopNodeId for vector store which has fielUpload + const vsNodesWithFileUpload = vsNodes.filter((node) => node.data.inputs?.fileUpload) + if (vsNodesWithFileUpload.length > 1) { + throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, 'Multiple vector store nodes with fileUpload enabled') + } else if (vsNodesWithFileUpload.length === 1 && !stopNodeId) { + stopNodeId = vsNodesWithFileUpload[0].data.id + } // Check if multiple vector store nodes exist, and if stopNodeId is specified if (vsNodes.length > 1 && !stopNodeId) { @@ -138,7 +145,7 @@ export const upsertVector = async (req: Request, isInternal: boolean = false) => const startingNodes = nodes.filter((nd) => startingNodeIds.includes(nd.data.id)) - await appServer.chatflowPool.add(chatflowid, undefined, startingNodes, incomingInput?.overrideConfig) + await appServer.chatflowPool.add(chatflowid, undefined, startingNodes, incomingInput?.overrideConfig, chatId) // Save to DB if (upsertedResult['flowData'] && upsertedResult['result']) { diff --git a/packages/ui/src/api/vectorstore.js b/packages/ui/src/api/vectorstore.js index 64f2ce65..54d1f4c0 100644 --- a/packages/ui/src/api/vectorstore.js +++ b/packages/ui/src/api/vectorstore.js @@ -1,11 +1,16 @@ import client from './client' const upsertVectorStore = (id, input) => client.post(`/vector/internal-upsert/${id}`, input) +const upsertVectorStoreWithFormData = (id, formData) => + client.post(`/vector/internal-upsert/${id}`, formData, { + headers: { 'Content-Type': 'multipart/form-data' } + }) const getUpsertHistory = (id, params = {}) => client.get(`/upsert-history/${id}`, { params: { order: 'DESC', ...params } }) const deleteUpsertHistory = (ids) => client.patch(`/upsert-history`, { ids }) export default { getUpsertHistory, upsertVectorStore, + upsertVectorStoreWithFormData, deleteUpsertHistory } diff --git a/packages/ui/src/ui-component/dialog/ViewMessagesDialog.jsx b/packages/ui/src/ui-component/dialog/ViewMessagesDialog.jsx index 7dcb98ad..d910787e 100644 --- a/packages/ui/src/ui-component/dialog/ViewMessagesDialog.jsx +++ b/packages/ui/src/ui-component/dialog/ViewMessagesDialog.jsx @@ -34,7 +34,7 @@ import userPNG from '@/assets/images/account.png' import msgEmptySVG from '@/assets/images/message_empty.svg' import multiagent_supervisorPNG from '@/assets/images/multiagent_supervisor.png' import multiagent_workerPNG from '@/assets/images/multiagent_worker.png' -import { IconTool, IconDeviceSdCard, IconFileExport, IconEraser, IconX, IconDownload } from '@tabler/icons-react' +import { IconTool, IconDeviceSdCard, IconFileExport, IconEraser, IconX, IconDownload, IconPaperclip } from '@tabler/icons-react' // Project import import { MemoizedReactMarkdown } from '@/ui-component/markdown/MemoizedReactMarkdown' @@ -438,6 +438,59 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => { setSourceDialogOpen(true) } + const renderFileUploads = (item, index) => { + if (item?.mime?.startsWith('image/')) { + return ( + + + + ) + } else if (item?.mime?.startsWith('audio/')) { + return ( + /* eslint-disable jsx-a11y/media-has-caption */ + + ) + } else { + return ( + + + + {item.name} + + + ) + } + } + useEffect(() => { const leadEmailFromChatMessages = chatMessages.filter((message) => message.type === 'userMessage' && message.leadEmail) if (leadEmailFromChatMessages.length) { @@ -855,37 +908,7 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => { }} > {message.fileUploads.map((item, index) => { - return ( - <> - {item.mime.startsWith('image/') ? ( - - - - ) : ( - // eslint-disable-next-line jsx-a11y/media-has-caption - - )} - - ) + return <>{renderFileUploads(item, index)} })} )} diff --git a/packages/ui/src/views/chatmessage/ChatMessage.jsx b/packages/ui/src/views/chatmessage/ChatMessage.jsx index 3fe42bd0..41a5fe15 100644 --- a/packages/ui/src/views/chatmessage/ChatMessage.jsx +++ b/packages/ui/src/views/chatmessage/ChatMessage.jsx @@ -37,7 +37,8 @@ import { IconTool, IconSquareFilled, IconDeviceSdCard, - IconCheck + IconCheck, + IconPaperclip } from '@tabler/icons-react' import robotPNG from '@/assets/images/robot.png' import userPNG from '@/assets/images/account.png' @@ -64,6 +65,7 @@ import './ChatMessage.css' import chatmessageApi from '@/api/chatmessage' import chatflowsApi from '@/api/chatflows' import predictionApi from '@/api/prediction' +import vectorstoreApi from '@/api/vectorstore' import chatmessagefeedbackApi from '@/api/chatmessagefeedback' import leadsApi from '@/api/lead' @@ -84,6 +86,71 @@ const messageImageStyle = { objectFit: 'cover' } +const CardWithDeleteOverlay = ({ item, customization, onDelete }) => { + const [isHovered, setIsHovered] = useState(false) + const defaultBackgroundColor = customization.isDarkMode ? 'rgba(0, 0, 0, 0.3)' : 'transparent' + + return ( +
setIsHovered(true)} + onMouseLeave={() => setIsHovered(false)} + style={{ position: 'relative', display: 'inline-block' }} + > + + + + {item.name} + + + {isHovered && ( + + )} +
+ ) +} + +CardWithDeleteOverlay.propTypes = { + item: PropTypes.object, + customization: PropTypes.object, + onDelete: PropTypes.func +} + export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, previews, setPreviews }) => { const theme = useTheme() const customization = useSelector((state) => state.customization) @@ -111,6 +178,9 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview const [sourceDialogProps, setSourceDialogProps] = useState({}) const [chatId, setChatId] = useState(uuidv4()) const [isMessageStopping, setIsMessageStopping] = useState(false) + const [uploadedFiles, setUploadedFiles] = useState([]) + const [imageUploadAllowedTypes, setImageUploadAllowedTypes] = useState('') + const [fileUploadAllowedTypes, setFileUploadAllowedTypes] = useState('') const inputRef = useRef(null) const getChatmessageApi = useApi(chatmessageApi.getInternalChatmessageFromChatflow) @@ -134,8 +204,10 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview const [isLeadSaved, setIsLeadSaved] = useState(false) // drag & drop and file input + const imgUploadRef = useRef(null) const fileUploadRef = useRef(null) - const [isChatFlowAvailableForUploads, setIsChatFlowAvailableForUploads] = useState(false) + const [isChatFlowAvailableForImageUploads, setIsChatFlowAvailableForImageUploads] = useState(false) + const [isChatFlowAvailableForFileUploads, setIsChatFlowAvailableForFileUploads] = useState(false) const [isDragActive, setIsDragActive] = useState(false) // recording @@ -158,6 +230,18 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview } }) } + if (constraints.isFileUploadAllowed) { + const fileExt = file.name.split('.').pop() + if (fileExt) { + constraints.fileUploadSizeAndTypes.map((allowed) => { + if (allowed.fileTypes.length === 1 && allowed.fileTypes[0] === '*') { + acceptFile = true + } else if (allowed.fileTypes.includes(`.${fileExt}`)) { + acceptFile = true + } + }) + } + } if (!acceptFile) { alert(`Cannot upload file. Kindly check the allowed file types and maximum allowed size.`) } @@ -165,12 +249,14 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview } const handleDrop = async (e) => { - if (!isChatFlowAvailableForUploads) { + if (!isChatFlowAvailableForImageUploads && !isChatFlowAvailableForFileUploads) { return } e.preventDefault() setIsDragActive(false) let files = [] + let uploadedFiles = [] + if (e.dataTransfer.files.length > 0) { for (const file of e.dataTransfer.files) { if (isFileAllowedForUpload(file) === false) { @@ -178,6 +264,7 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview } const reader = new FileReader() const { name } = file + uploadedFiles.push(file) files.push( new Promise((resolve) => { reader.onload = (evt) => { @@ -188,7 +275,7 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview let previewUrl if (file.type.startsWith('audio/')) { previewUrl = audioUploadSVG - } else if (file.type.startsWith('image/')) { + } else { previewUrl = URL.createObjectURL(file) } resolve({ @@ -205,10 +292,12 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview } const newFiles = await Promise.all(files) + setUploadedFiles(uploadedFiles) setPreviews((prevPreviews) => [...prevPreviews, ...newFiles]) } if (e.dataTransfer.items) { + //TODO set files for (const item of e.dataTransfer.items) { if (item.kind === 'string' && item.type.match('^text/uri-list')) { item.getAsString((s) => { @@ -246,10 +335,12 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview return } let files = [] + let uploadedFiles = [] for (const file of event.target.files) { if (isFileAllowedForUpload(file) === false) { return } + uploadedFiles.push(file) const reader = new FileReader() const { name } = file files.push( @@ -273,6 +364,7 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview } const newFiles = await Promise.all(files) + setUploadedFiles(uploadedFiles) setPreviews((prevPreviews) => [...prevPreviews, ...newFiles]) // 👇️ reset file input event.target.value = null @@ -303,7 +395,7 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview } const handleDrag = (e) => { - if (isChatFlowAvailableForUploads) { + if (isChatFlowAvailableForImageUploads || isChatFlowAvailableForFileUploads) { e.preventDefault() e.stopPropagation() if (e.type === 'dragenter' || e.type === 'dragover') { @@ -343,11 +435,16 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview setPreviews(previews.filter((item) => item !== itemToDelete)) } - const handleUploadClick = () => { + const handleFileUploadClick = () => { // 👇️ open file input box on click of another element fileUploadRef.current.click() } + const handleImageUploadClick = () => { + // 👇️ open file input box on click of another element + imgUploadRef.current.click() + } + const clearPreviews = () => { // Revoke the data uris to avoid memory leaks previews.forEach((file) => URL.revokeObjectURL(file.preview)) @@ -489,6 +586,7 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview setMessages((prevMessages) => [...prevMessages, { message, type: 'apiMessage' }]) setLoading(false) setUserInput('') + setUploadedFiles([]) setTimeout(() => { inputRef.current?.focus() }, 100) @@ -526,7 +624,7 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview if (selectedInput !== undefined && selectedInput.trim() !== '') input = selectedInput setLoading(true) - const urls = previews.map((item) => { + const uploads = previews.map((item) => { return { data: item.data, type: item.type, @@ -535,7 +633,7 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview } }) clearPreviews() - setMessages((prevMessages) => [...prevMessages, { message: input, type: 'userMessage', fileUploads: urls }]) + setMessages((prevMessages) => [...prevMessages, { message: input, type: 'userMessage', fileUploads: uploads }]) // Send user question to Prediction Internal API try { @@ -543,11 +641,30 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview question: input, chatId } - if (urls && urls.length > 0) params.uploads = urls + if (uploads && uploads.length > 0) params.uploads = uploads if (leadEmail) params.leadEmail = leadEmail if (isChatFlowAvailableToStream) params.socketIOClientId = socketIOClientId if (action) params.action = action + if (uploadedFiles.length > 0) { + const formData = new FormData() + for (const file of uploadedFiles) { + formData.append('files', file) + } + formData.append('chatId', chatId) + + const response = await vectorstoreApi.upsertVectorStoreWithFormData(chatflowid, formData) + if (!response.data) { + setMessages((prevMessages) => [...prevMessages, { message: 'Unable to upload documents', type: 'apiMessage' }]) + } else { + // delay for vector store to be updated + const delay = (delayInms) => { + return new Promise((resolve) => setTimeout(resolve, delayInms)) + } + await delay(2500) //TODO: check if embeddings can be retrieved using file name as metadata filter + } + } + const response = await predictionApi.sendMessageAndGetPrediction(chatflowid, params) if (response.data) { @@ -598,6 +715,7 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview setLocalStorageChatflow(chatflowid, data.chatId) setLoading(false) setUserInput('') + setUploadedFiles([]) setTimeout(() => { inputRef.current?.focus() scrollToBottom() @@ -717,8 +835,11 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview // Get chatflow uploads capability useEffect(() => { if (getAllowChatFlowUploads.data) { - setIsChatFlowAvailableForUploads(getAllowChatFlowUploads.data?.isImageUploadAllowed ?? false) + setIsChatFlowAvailableForImageUploads(getAllowChatFlowUploads.data?.isImageUploadAllowed ?? false) + setIsChatFlowAvailableForFileUploads(getAllowChatFlowUploads.data?.isFileUploadAllowed ?? false) setIsChatFlowAvailableForSpeech(getAllowChatFlowUploads.data?.isSpeechToTextEnabled ?? false) + setImageUploadAllowedTypes(getAllowChatFlowUploads.data?.imgUploadSizeAndTypes.map((allowed) => allowed.fileTypes).join(',')) + setFileUploadAllowedTypes(getAllowChatFlowUploads.data?.fileUploadSizeAndTypes.map((allowed) => allowed.fileTypes).join(',')) } // eslint-disable-next-line react-hooks/exhaustive-deps }, [getAllowChatFlowUploads.data]) @@ -822,6 +943,7 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview return () => { setUserInput('') + setUploadedFiles([]) setLoading(false) setMessages([ { @@ -965,6 +1087,105 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview ) } + const previewDisplay = (item) => { + if (item.mime.startsWith('image/')) { + return ( + handleDeletePreview(item)} + > + + + + + + + ) + } else if (item.mime.startsWith('audio/')) { + return ( + + + handleDeletePreview(item)} size='small'> + + + + ) + } else { + return handleDeletePreview(item)} /> + } + } + + const renderFileUploads = (item, index) => { + if (item?.mime?.startsWith('image/')) { + return ( + + + + ) + } else if (item?.mime?.startsWith('audio/')) { + return ( + /* eslint-disable jsx-a11y/media-has-caption */ + + ) + } else { + return ( + + + + {item.name} + + + ) + } + } + return (
{isDragActive && ( @@ -976,19 +1197,25 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview onDrop={handleDrop} /> )} - {isDragActive && getAllowChatFlowUploads.data?.isImageUploadAllowed && ( - - Drop here to upload - {getAllowChatFlowUploads.data.imgUploadSizeAndTypes.map((allowed) => { - return ( - <> - {allowed.fileTypes?.join(', ')} - Max Allowed Size: {allowed.maxUploadSize} MB - - ) - })} - - )} + {isDragActive && + (getAllowChatFlowUploads.data?.isImageUploadAllowed || getAllowChatFlowUploads.data?.isFileAllowedForUpload) && ( + + Drop here to upload + {[ + ...getAllowChatFlowUploads.data.imgUploadSizeAndTypes, + ...getAllowChatFlowUploads.data.fileUploadSizeAndTypes + ].map((allowed) => { + return ( + <> + {allowed.fileTypes?.join(', ')} + {allowed.maxUploadSize && ( + Max Allowed Size: {allowed.maxUploadSize} MB + )} + + ) + })} + + )}
{messages && @@ -1038,36 +1265,7 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview }} > {message.fileUploads.map((item, index) => { - return ( - <> - {item?.mime?.startsWith('image/') ? ( - - - - ) : ( - // eslint-disable-next-line jsx-a11y/media-has-caption - - )} - - ) + return <>{renderFileUploads(item, index)} })}
)} @@ -1564,45 +1762,7 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview {previews && previews.length > 0 && ( {previews.map((item, index) => ( - - {item.mime.startsWith('image/') ? ( - handleDeletePreview(item)} - > - - - - - - - ) : ( - - - handleDeletePreview(item)} size='small'> - - - - )} - + {previewDisplay(item)} ))} )} @@ -1679,15 +1839,62 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview multiline={true} maxRows={isDialog ? 7 : 2} startAdornment={ - isChatFlowAvailableForUploads && ( - - - - - - ) + <> + {isChatFlowAvailableForImageUploads && !isChatFlowAvailableForFileUploads && ( + + + + + + )} + {!isChatFlowAvailableForImageUploads && isChatFlowAvailableForFileUploads && ( + + + + + + )} + {isChatFlowAvailableForImageUploads && isChatFlowAvailableForFileUploads && ( + + + + + + + + + )} + {!isChatFlowAvailableForImageUploads && !isChatFlowAvailableForFileUploads && } + } endAdornment={ <> @@ -1707,7 +1914,7 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview )} {!isAgentCanvas && ( - + {loading ? (
@@ -1727,7 +1934,7 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview {isAgentCanvas && ( <> {!loading && ( - + } /> - {isChatFlowAvailableForUploads && ( - + {isChatFlowAvailableForImageUploads && ( + + )} + {isChatFlowAvailableForFileUploads && ( + )} )}