Fix merge conflicts

CONTRIBUTING-ZH.md

@@ -125,6 +125,7 @@ Flowise 支持不同的环境变量来配置您的实例。您可以在 `package
 | FLOWISE_PASSWORD            | 登录密码                                               | 字符串                                          |                                     |
 | FLOWISE_FILE_SIZE_LIMIT     | 上传文件大小限制                                       | 字符串                                          | 50mb                                |
 | DEBUG                       | 打印组件的日志                                         | 布尔值                                          |                                     |
+| BLOB_STORAGE_PATH           | 存储位置                                               | 字符串                                          | `your-home-dir/.flowise/storage`    |
 | LOG_PATH                    | 存储日志文件的位置                                     | 字符串                                          | `your-path/Flowise/logs`            |
 | LOG_LEVEL                   | 日志的不同级别                                         | 枚举字符串: `error`, `info`, `verbose`, `debug` | `info`                              |
 | APIKEY_PATH                 | 存储 API 密钥的位置                                    | 字符串                                          | `your-path/Flowise/packages/server` |

CONTRIBUTING.md

@@ -129,6 +129,7 @@ Flowise support different environment variables to configure your instance. You
 | FLOWISE_PASSWORD            | Password to login                                                            | String                                           |                                     |
 | FLOWISE_FILE_SIZE_LIMIT     | Upload File Size Limit                                                       | String                                           | 50mb                                |
 | DEBUG                       | Print logs from components                                                   | Boolean                                          |                                     |
+| BLOB_STORAGE_PATH           | Location where uploaded files are stored                                     | String                                           | `your-home-dir/.flowise/storage`    |
 | LOG_PATH                    | Location where log files are stored                                          | String                                           | `your-path/Flowise/logs`            |
 | LOG_LEVEL                   | Different levels of logs                                                     | Enum String: `error`, `info`, `verbose`, `debug` | `info`                              |
 | APIKEY_PATH                 | Location where api keys are saved                                            | String                                           | `your-path/Flowise/packages/server` |

docker/.env.example

@@ -3,6 +3,7 @@ DATABASE_PATH=/root/.flowise
 APIKEY_PATH=/root/.flowise
 SECRETKEY_PATH=/root/.flowise
 LOG_PATH=/root/.flowise/logs
+BLOB_STORAGE_PATH=/root/.flowise/storage
 
 # CORS_ORIGINS="*"
 # IFRAME_ORIGINS="*"

docker/README.md

@@ -31,5 +31,6 @@ If you like to persist your data (flows, logs, apikeys, credentials), set these
 -   APIKEY_PATH=/root/.flowise
 -   LOG_PATH=/root/.flowise/logs
 -   SECRETKEY_PATH=/root/.flowise
+-   BLOB_STORAGE_PATH=/root/.flowise/storage
 
 Flowise also support different environment variables to configure your instance. Read [more](https://docs.flowiseai.com/environment-variables)

docker/docker-compose.yml

@@ -26,6 +26,7 @@ services:
             - FLOWISE_SECRETKEY_OVERWRITE=${FLOWISE_SECRETKEY_OVERWRITE}
             - LOG_LEVEL=${LOG_LEVEL}
             - LOG_PATH=${LOG_PATH}
+            - BLOB_STORAGE_PATH=${BLOB_STORAGE_PATH}
             - DISABLE_FLOWISE_TELEMETRY=${DISABLE_FLOWISE_TELEMETRY}
         ports:
             - '${PORT}:${PORT}'

package.json

@@ -1,6 +1,6 @@
 {
     "name": "flowise",
-    "version": "1.5.0",
+    "version": "1.5.1",
     "private": true,
     "homepage": "https://flowiseai.com",
     "workspaces": [

packages/components/credentials/AssemblyAI.credential.ts

@@ -0,0 +1,23 @@
+import { INodeParams, INodeCredential } from '../src/Interface'
+
+class AssemblyAIApi implements INodeCredential {
+    label: string
+    name: string
+    version: number
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'AssemblyAI API'
+        this.name = 'assemblyAIApi'
+        this.version = 1.0
+        this.inputs = [
+            {
+                label: 'AssemblyAI Api Key',
+                name: 'assemblyAIApiKey',
+                type: 'password'
+            }
+        ]
+    }
+}
+
+module.exports = { credClass: AssemblyAIApi }

packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts

@@ -4,13 +4,15 @@ import { BaseChatModel } from '@langchain/core/language_models/chat_models'
 import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages'
 import { ChainValues } from '@langchain/core/utils/types'
 import { AgentStep } from '@langchain/core/agents'
-import { renderTemplate } from '@langchain/core/prompts'
+import { renderTemplate, MessagesPlaceholder } from '@langchain/core/prompts'
 import { RunnableSequence } from '@langchain/core/runnables'
 import { ChatConversationalAgent } from 'langchain/agents'
 import { getBaseClasses } from '../../../src/utils'
 import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
 import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { AgentExecutor } from '../../../src/agents'
+import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
+import { addImagesToMessages } from '../../../src/multiModalUtils'
 
 const DEFAULT_PREFIX = `Assistant is a large language model trained by OpenAI.
 
@@ -81,12 +83,18 @@ class ConversationalAgent_Agents implements INode {
     }
 
     async init(nodeData: INodeData, input: string, options: ICommonObject): Promise<any> {
-        return prepareAgent(nodeData, { sessionId: this.sessionId, chatId: options.chatId, input }, options.chatHistory)
+        return prepareAgent(nodeData, options, { sessionId: this.sessionId, chatId: options.chatId, input }, options.chatHistory)
     }
 
     async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string> {
         const memory = nodeData.inputs?.memory as FlowiseMemory
-        const executor = await prepareAgent(nodeData, { sessionId: this.sessionId, chatId: options.chatId, input }, options.chatHistory)
+
+        const executor = await prepareAgent(
+            nodeData,
+            options,
+            { sessionId: this.sessionId, chatId: options.chatId, input },
+            options.chatHistory
+        )
 
         const loggerHandler = new ConsoleCallbackHandler(options.logger)
         const callbacks = await additionalCallbacks(nodeData, options)
@@ -120,6 +128,7 @@ class ConversationalAgent_Agents implements INode {
 
 const prepareAgent = async (
     nodeData: INodeData,
+    options: ICommonObject,
     flowObj: { sessionId?: string; chatId?: string; input?: string },
     chatHistory: IMessage[] = []
 ) => {
@@ -131,11 +140,6 @@ const prepareAgent = async (
     const memoryKey = memory.memoryKey ? memory.memoryKey : 'chat_history'
     const inputKey = memory.inputKey ? memory.inputKey : 'input'
 
-    /** Bind a stop token to the model */
-    const modelWithStop = model.bind({
-        stop: ['\nObservation']
-    })
-
     const outputParser = ChatConversationalAgent.getDefaultOutputParser({
         llm: model,
         toolNames: tools.map((tool) => tool.name)
@@ -146,6 +150,41 @@ const prepareAgent = async (
         outputParser
     })
 
+    if (model instanceof ChatOpenAI) {
+        let humanImageMessages: HumanMessage[] = []
+        const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
+
+        if (messageContent?.length) {
+            // Change model to gpt-4-vision
+            model.modelName = 'gpt-4-vision-preview'
+
+            // Change default max token to higher when using gpt-4-vision
+            model.maxTokens = 1024
+
+            for (const msg of messageContent) {
+                humanImageMessages.push(new HumanMessage({ content: [msg] }))
+            }
+
+            // Pop the `agent_scratchpad` MessagePlaceHolder
+            let messagePlaceholder = prompt.promptMessages.pop() as MessagesPlaceholder
+
+            // Add the HumanMessage for images
+            prompt.promptMessages.push(...humanImageMessages)
+
+            // Add the `agent_scratchpad` MessagePlaceHolder back
+            prompt.promptMessages.push(messagePlaceholder)
+        } else {
+            // revert to previous values if image upload is empty
+            model.modelName = model.configuredModel
+            model.maxTokens = model.configuredMaxToken
+        }
+    }
+
+    /** Bind a stop token to the model */
+    const modelWithStop = model.bind({
+        stop: ['\nObservation']
+    })
+
     const runnableAgent = RunnableSequence.from([
         {
             [inputKey]: (i: { input: string; steps: AgentStep[] }) => i.input,
@@ -166,7 +205,7 @@ const prepareAgent = async (
         sessionId: flowObj?.sessionId,
         chatId: flowObj?.chatId,
         input: flowObj?.input,
-        verbose: process.env.DEBUG === 'true' ? true : false
+        verbose: process.env.DEBUG === 'true'
     })
 
     return executor

packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts

@@ -1,12 +1,17 @@
 import { flatten } from 'lodash'
-import { AgentExecutor, createReactAgent } from 'langchain/agents'
+import { AgentExecutor } from 'langchain/agents'
 import { pull } from 'langchain/hub'
 import { Tool } from '@langchain/core/tools'
 import type { PromptTemplate } from '@langchain/core/prompts'
 import { BaseChatModel } from '@langchain/core/language_models/chat_models'
 import { additionalCallbacks } from '../../../src/handler'
-import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { getBaseClasses } from '../../../src/utils'
+import { createReactAgent } from '../../../src/agents'
+import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
+import { HumanMessage } from '@langchain/core/messages'
+import { addImagesToMessages } from '../../../src/multiModalUtils'
+import { ChatPromptTemplate, HumanMessagePromptTemplate } from 'langchain/prompts'
 
 class MRKLAgentChat_Agents implements INode {
     label: string
@@ -18,11 +23,12 @@ class MRKLAgentChat_Agents implements INode {
     category: string
     baseClasses: string[]
     inputs: INodeParams[]
+    sessionId?: string
 
-    constructor() {
+    constructor(fields?: { sessionId?: string }) {
         this.label = 'ReAct Agent for Chat Models'
         this.name = 'mrklAgentChat'
-        this.version = 2.0
+        this.version = 3.0
         this.type = 'AgentExecutor'
         this.category = 'Agents'
         this.icon = 'agent.svg'
@@ -39,8 +45,14 @@ class MRKLAgentChat_Agents implements INode {
                 label: 'Chat Model',
                 name: 'model',
                 type: 'BaseChatModel'
+            },
+            {
+                label: 'Memory',
+                name: 'memory',
+                type: 'BaseChatMemory'
             }
         ]
+        this.sessionId = fields?.sessionId
     }
 
     async init(): Promise<any> {
@@ -48,30 +60,67 @@ class MRKLAgentChat_Agents implements INode {
     }
 
     async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string> {
+        const memory = nodeData.inputs?.memory as FlowiseMemory
         const model = nodeData.inputs?.model as BaseChatModel
         let tools = nodeData.inputs?.tools as Tool[]
         tools = flatten(tools)
 
-        const promptWithChat = await pull<PromptTemplate>('hwchase17/react-chat')
+        const prompt = await pull<PromptTemplate>('hwchase17/react-chat')
+        let chatPromptTemplate = undefined
+
+        if (model instanceof ChatOpenAI) {
+            const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
+
+            if (messageContent?.length) {
+                // Change model to gpt-4-vision
+                model.modelName = 'gpt-4-vision-preview'
+
+                // Change default max token to higher when using gpt-4-vision
+                model.maxTokens = 1024
+
+                const oldTemplate = prompt.template as string
+                chatPromptTemplate = ChatPromptTemplate.fromMessages([HumanMessagePromptTemplate.fromTemplate(oldTemplate)])
+                chatPromptTemplate.promptMessages.push(new HumanMessage({ content: messageContent }))
+            } else {
+                // revert to previous values if image upload is empty
+                model.modelName = model.configuredModel
+                model.maxTokens = model.configuredMaxToken
+            }
+        }
 
         const agent = await createReactAgent({
             llm: model,
             tools,
-            prompt: promptWithChat
+            prompt: chatPromptTemplate ?? prompt
         })
 
         const executor = new AgentExecutor({
             agent,
             tools,
-            verbose: process.env.DEBUG === 'true' ? true : false
+            verbose: process.env.DEBUG === 'true'
         })
 
         const callbacks = await additionalCallbacks(nodeData, options)
 
-        const result = await executor.invoke({
-            input,
-            callbacks
-        })
+        const prevChatHistory = options.chatHistory
+        const chatHistory = ((await memory.getChatMessages(this.sessionId, false, prevChatHistory)) as IMessage[]) ?? []
+        const chatHistoryString = chatHistory.map((hist) => hist.message).join('\\n')
+
+        const result = await executor.invoke({ input, chat_history: chatHistoryString }, { callbacks })
+
+        await memory.addChatMessages(
+            [
+                {
+                    text: input,
+                    type: 'userMessage'
+                },
+                {
+                    text: result?.output,
+                    type: 'apiMessage'
+                }
+            ],
+            this.sessionId
+        )
 
         return result?.output
     }

packages/components/nodes/agents/MRKLAgentLLM/MRKLAgentLLM.ts

@@ -1,5 +1,5 @@
 import { flatten } from 'lodash'
-import { AgentExecutor, createReactAgent } from 'langchain/agents'
+import { AgentExecutor } from 'langchain/agents'
 import { pull } from 'langchain/hub'
 import { Tool } from '@langchain/core/tools'
 import type { PromptTemplate } from '@langchain/core/prompts'
@@ -7,6 +7,7 @@ import { BaseLanguageModel } from 'langchain/base_language'
 import { additionalCallbacks } from '../../../src/handler'
 import { getBaseClasses } from '../../../src/utils'
 import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { createReactAgent } from '../../../src/agents'
 
 class MRKLAgentLLM_Agents implements INode {
     label: string
@@ -68,10 +69,7 @@ class MRKLAgentLLM_Agents implements INode {
 
         const callbacks = await additionalCallbacks(nodeData, options)
 
-        const result = await executor.invoke({
-            input,
-            callbacks
-        })
+        const result = await executor.invoke({ input }, { callbacks })
 
         return result?.output
     }

packages/components/nodes/chains/ConversationChain/ConversationChain.ts

@@ -1,14 +1,16 @@
+import { ConversationChain } from 'langchain/chains'
 import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder, SystemMessagePromptTemplate } from '@langchain/core/prompts'
-import { BaseChatModel } from '@langchain/core/language_models/chat_models'
 import { RunnableSequence } from '@langchain/core/runnables'
 import { StringOutputParser } from '@langchain/core/output_parsers'
+import { HumanMessage } from '@langchain/core/messages'
 import { ConsoleCallbackHandler as LCConsoleCallbackHandler } from '@langchain/core/tracers/console'
-import { ConversationChain } from 'langchain/chains'
-import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface'
-import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
-import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils'
 import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation'
 import { formatResponse } from '../../outputparsers/OutputParserHelpers'
+import { addImagesToMessages } from '../../../src/multiModalUtils'
+import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
+import { FlowiseMemory, ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
+import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils'
 
 let systemMessage = `The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.`
 const inputKey = 'input'
@@ -86,12 +88,14 @@ class ConversationChain_Chains implements INode {
     }
 
     async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
-        const chain = prepareChain(nodeData, this.sessionId, options.chatHistory)
+        const chain = prepareChain(nodeData, options, this.sessionId)
         return chain
     }
 
     async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string | object> {
         const memory = nodeData.inputs?.memory
+
+        const chain = prepareChain(nodeData, options, this.sessionId)
         const moderations = nodeData.inputs?.inputModeration as Moderation[]
 
         if (moderations && moderations.length > 0) {
@@ -105,8 +109,6 @@ class ConversationChain_Chains implements INode {
             }
         }
 
-        const chain = prepareChain(nodeData, this.sessionId, options.chatHistory)
-
         const loggerHandler = new ConsoleCallbackHandler(options.logger)
         const additionalCallback = await additionalCallbacks(nodeData, options)
 
@@ -143,7 +145,7 @@ class ConversationChain_Chains implements INode {
     }
 }
 
-const prepareChatPrompt = (nodeData: INodeData) => {
+const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: HumanMessage[]) => {
     const memory = nodeData.inputs?.memory as FlowiseMemory
     const prompt = nodeData.inputs?.systemMessagePrompt as string
     const chatPromptTemplate = nodeData.inputs?.chatPromptTemplate as ChatPromptTemplate
@@ -151,12 +153,10 @@ const prepareChatPrompt = (nodeData: INodeData) => {
     if (chatPromptTemplate && chatPromptTemplate.promptMessages.length) {
         const sysPrompt = chatPromptTemplate.promptMessages[0]
         const humanPrompt = chatPromptTemplate.promptMessages[chatPromptTemplate.promptMessages.length - 1]
-        const chatPrompt = ChatPromptTemplate.fromMessages([
-            sysPrompt,
-            new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'),
-            humanPrompt
-        ])
+        const messages = [sysPrompt, new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'), humanPrompt]
+        if (humanImageMessages.length) messages.push(...humanImageMessages)
 
+        const chatPrompt = ChatPromptTemplate.fromMessages(messages)
         if ((chatPromptTemplate as any).promptValues) {
             // @ts-ignore
             chatPrompt.promptValues = (chatPromptTemplate as any).promptValues
@@ -165,21 +165,46 @@ const prepareChatPrompt = (nodeData: INodeData) => {
         return chatPrompt
     }
 
-    const chatPrompt = ChatPromptTemplate.fromMessages([
+    const messages = [
         SystemMessagePromptTemplate.fromTemplate(prompt ? prompt : systemMessage),
         new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'),
         HumanMessagePromptTemplate.fromTemplate(`{${inputKey}}`)
-    ])
+    ]
+    if (humanImageMessages.length) messages.push(...(humanImageMessages as any[]))
+
+    const chatPrompt = ChatPromptTemplate.fromMessages(messages)
 
     return chatPrompt
 }
 
-const prepareChain = (nodeData: INodeData, sessionId?: string, chatHistory: IMessage[] = []) => {
-    const model = nodeData.inputs?.model as BaseChatModel
+const prepareChain = (nodeData: INodeData, options: ICommonObject, sessionId?: string) => {
+    const chatHistory = options.chatHistory
+    let model = nodeData.inputs?.model as ChatOpenAI
     const memory = nodeData.inputs?.memory as FlowiseMemory
     const memoryKey = memory.memoryKey ?? 'chat_history'
 
-    const chatPrompt = prepareChatPrompt(nodeData)
+    let humanImageMessages: HumanMessage[] = []
+    if (model instanceof ChatOpenAI) {
+        const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
+
+        if (messageContent?.length) {
+            // Change model to gpt-4-vision
+            model.modelName = 'gpt-4-vision-preview'
+
+            // Change default max token to higher when using gpt-4-vision
+            model.maxTokens = 1024
+
+            for (const msg of messageContent) {
+                humanImageMessages.push(new HumanMessage({ content: [msg] }))
+            }
+        } else {
+            // revert to previous values if image upload is empty
+            model.modelName = model.configuredModel
+            model.maxTokens = model.configuredMaxToken
+        }
+    }
+
+    const chatPrompt = prepareChatPrompt(nodeData, humanImageMessages)
     let promptVariables = {}
     const promptValuesRaw = (chatPrompt as any).promptValues
     if (promptValuesRaw) {
@@ -203,7 +228,7 @@ const prepareChain = (nodeData: INodeData, sessionId?: string, chatHistory: IMes
             },
             ...promptVariables
         },
-        chatPrompt,
+        prepareChatPrompt(nodeData, humanImageMessages),
         model,
         new StringOutputParser()
     ])

packages/components/nodes/chains/LLMChain/LLMChain.ts

@@ -1,5 +1,6 @@
 import { BaseLanguageModel, BaseLanguageModelCallOptions } from '@langchain/core/language_models/base'
 import { BaseLLMOutputParser, BaseOutputParser } from '@langchain/core/output_parsers'
+import { ChatPromptTemplate, FewShotPromptTemplate, PromptTemplate, HumanMessagePromptTemplate } from '@langchain/core/prompts'
 import { OutputFixingParser } from 'langchain/output_parsers'
 import { LLMChain } from 'langchain/chains'
 import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
@@ -7,6 +8,9 @@ import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from
 import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils'
 import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation'
 import { formatResponse, injectOutputParser } from '../../outputparsers/OutputParserHelpers'
+import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
+import { addImagesToMessages } from '../../../src/multiModalUtils'
+import { HumanMessage } from 'langchain/schema'
 
 class LLMChain_Chains implements INode {
     label: string
@@ -160,12 +164,7 @@ const runPrediction = async (
     const socketIO = isStreaming ? options.socketIO : undefined
     const socketIOClientId = isStreaming ? options.socketIOClientId : ''
     const moderations = nodeData.inputs?.inputModeration as Moderation[]
-    /**
-     * Apply string transformation to reverse converted special chars:
-     * FROM: { "value": "hello i am benFLOWISE_NEWLINEFLOWISE_NEWLINEFLOWISE_TABhow are you?" }
-     * TO: { "value": "hello i am ben\n\n\thow are you?" }
-     */
-    const promptValues = handleEscapeCharacters(promptValuesRaw, true)
+    let model = nodeData.inputs?.model as ChatOpenAI
 
     if (moderations && moderations.length > 0) {
         try {
@@ -178,6 +177,46 @@ const runPrediction = async (
         }
     }
 
+    /**
+     * Apply string transformation to reverse converted special chars:
+     * FROM: { "value": "hello i am benFLOWISE_NEWLINEFLOWISE_NEWLINEFLOWISE_TABhow are you?" }
+     * TO: { "value": "hello i am ben\n\n\thow are you?" }
+     */
+    const promptValues = handleEscapeCharacters(promptValuesRaw, true)
+    const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
+
+    if (chain.llm instanceof ChatOpenAI) {
+        const chatOpenAI = chain.llm as ChatOpenAI
+        if (messageContent?.length) {
+            // Change model to gpt-4-vision && max token to higher when using gpt-4-vision
+            chatOpenAI.modelName = 'gpt-4-vision-preview'
+            chatOpenAI.maxTokens = 1024
+            // Add image to the message
+            if (chain.prompt instanceof PromptTemplate) {
+                const existingPromptTemplate = chain.prompt.template as string
+                let newChatPromptTemplate = ChatPromptTemplate.fromMessages([
+                    HumanMessagePromptTemplate.fromTemplate(existingPromptTemplate)
+                ])
+                newChatPromptTemplate.promptMessages.push(new HumanMessage({ content: messageContent }))
+                chain.prompt = newChatPromptTemplate
+            } else if (chain.prompt instanceof ChatPromptTemplate) {
+                chain.prompt.promptMessages.push(new HumanMessage({ content: messageContent }))
+            } else if (chain.prompt instanceof FewShotPromptTemplate) {
+                let existingFewShotPromptTemplate = chain.prompt.examplePrompt.template as string
+                let newFewShotPromptTemplate = ChatPromptTemplate.fromMessages([
+                    HumanMessagePromptTemplate.fromTemplate(existingFewShotPromptTemplate)
+                ])
+                newFewShotPromptTemplate.promptMessages.push(new HumanMessage({ content: messageContent }))
+                // @ts-ignore
+                chain.prompt.examplePrompt = newFewShotPromptTemplate
+            }
+        } else {
+            // revert to previous values if image upload is empty
+            chatOpenAI.modelName = model.configuredModel
+            chatOpenAI.maxTokens = model.configuredMaxToken
+        }
+    }
+
     if (promptValues && inputVariables.length > 0) {
         let seen: string[] = []
 

packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts

@@ -1,8 +1,10 @@
-import { ChatOpenAI, OpenAIChatInput } from '@langchain/openai'
+import type { ClientOptions } from 'openai'
+import { ChatOpenAI as LangchainChatOpenAI, OpenAIChatInput, AzureOpenAIInput, LegacyOpenAIInput } from '@langchain/openai'
 import { BaseCache } from '@langchain/core/caches'
-import { BaseLLMParams } from '@langchain/core/language_models/llms'
-import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { BaseChatModelParams } from '@langchain/core/language_models/chat_models'
+import { ICommonObject, IMultiModalOption, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { ChatOpenAI } from './FlowiseChatOpenAI'
 
 class ChatOpenAI_ChatModels implements INode {
     label: string
@@ -19,12 +21,12 @@ class ChatOpenAI_ChatModels implements INode {
     constructor() {
         this.label = 'ChatOpenAI'
         this.name = 'chatOpenAI'
-        this.version = 3.0
+        this.version = 5.0
         this.type = 'ChatOpenAI'
         this.icon = 'openai.svg'
         this.category = 'Chat Models'
         this.description = 'Wrapper around OpenAI large language models that use the Chat endpoint'
-        this.baseClasses = [this.type, ...getBaseClasses(ChatOpenAI)]
+        this.baseClasses = [this.type, ...getBaseClasses(LangchainChatOpenAI)]
         this.credential = {
             label: 'Connect Credential',
             name: 'credential',
@@ -168,6 +170,38 @@ class ChatOpenAI_ChatModels implements INode {
                 type: 'json',
                 optional: true,
                 additionalParams: true
+            },
+            {
+                label: 'Allow Image Uploads',
+                name: 'allowImageUploads',
+                type: 'boolean',
+                description:
+                    'Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent',
+                default: false,
+                optional: true
+            },
+            {
+                label: 'Image Resolution',
+                description: 'This parameter controls the resolution in which the model views the image.',
+                name: 'imageResolution',
+                type: 'options',
+                options: [
+                    {
+                        label: 'Low',
+                        name: 'low'
+                    },
+                    {
+                        label: 'High',
+                        name: 'high'
+                    },
+                    {
+                        label: 'Auto',
+                        name: 'auto'
+                    }
+                ],
+                default: 'low',
+                optional: false,
+                additionalParams: true
             }
         ]
     }
@@ -184,12 +218,17 @@ class ChatOpenAI_ChatModels implements INode {
         const basePath = nodeData.inputs?.basepath as string
         const baseOptions = nodeData.inputs?.baseOptions
 
+        const allowImageUploads = nodeData.inputs?.allowImageUploads as boolean
+        const imageResolution = nodeData.inputs?.imageResolution as string
+
         const credentialData = await getCredentialData(nodeData.credential ?? '', options)
         const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData)
 
         const cache = nodeData.inputs?.cache as BaseCache
 
-        const obj: Partial<OpenAIChatInput> & BaseLLMParams & { openAIApiKey?: string } = {
+        const obj: Partial<OpenAIChatInput> &
+            Partial<AzureOpenAIInput> &
+            BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput; multiModalOption?: IMultiModalOption } = {
             temperature: parseFloat(temperature),
             modelName,
             openAIApiKey,
@@ -212,10 +251,24 @@ class ChatOpenAI_ChatModels implements INode {
                 throw new Error("Invalid JSON in the ChatOpenAI's BaseOptions: " + exception)
             }
         }
-        const model = new ChatOpenAI(obj, {
-            basePath,
-            baseOptions: parsedBaseOptions
-        })
+
+        if (basePath || parsedBaseOptions) {
+            obj.configuration = {
+                baseURL: basePath,
+                baseOptions: parsedBaseOptions
+            }
+        }
+
+        const multiModalOption: IMultiModalOption = {
+            image: {
+                allowImageUploads: allowImageUploads ?? false,
+                imageResolution
+            }
+        }
+        obj.multiModalOption = multiModalOption
+
+        const model = new ChatOpenAI(nodeData.id, obj)
+
         return model
     }
 }

packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts

@@ -0,0 +1,38 @@
+import type { ClientOptions } from 'openai'
+import {
+    ChatOpenAI as LangchainChatOpenAI,
+    OpenAIChatInput,
+    LegacyOpenAIInput,
+    AzureOpenAIInput,
+    ChatOpenAICallOptions
+} from '@langchain/openai'
+import { BaseChatModelParams } from '@langchain/core/language_models/chat_models'
+import { IMultiModalOption } from '../../../src'
+import { BaseMessageLike, LLMResult } from 'langchain/schema'
+import { Callbacks } from '@langchain/core/callbacks/manager'
+
+export class ChatOpenAI extends LangchainChatOpenAI {
+    configuredModel: string
+    configuredMaxToken?: number
+    multiModalOption?: IMultiModalOption
+    id: string
+
+    constructor(
+        id: string,
+        fields?: Partial<OpenAIChatInput> &
+            Partial<AzureOpenAIInput> &
+            BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput; multiModalOption?: IMultiModalOption },
+        /** @deprecated */
+        configuration?: ClientOptions & LegacyOpenAIInput
+    ) {
+        super(fields, configuration)
+        this.id = id
+        this.multiModalOption = fields?.multiModalOption
+        this.configuredModel = fields?.modelName ?? 'gpt-3.5-turbo'
+        this.configuredMaxToken = fields?.maxTokens
+    }
+
+    async generate(messages: BaseMessageLike[][], options?: string[] | ChatOpenAICallOptions, callbacks?: Callbacks): Promise<LLMResult> {
+        return super.generate(messages, options, callbacks)
+    }
+}

packages/components/nodes/memory/UpstashRedisBackedChatMemory/UpstashRedisBackedChatMemory.ts

@@ -1,4 +1,5 @@
-import { Redis } from '@upstash/redis'
+import { Redis, RedisConfigNodejs } from '@upstash/redis'
+import { isEqual } from 'lodash'
 import { BufferMemory, BufferMemoryInput } from 'langchain/memory'
 import { UpstashRedisChatMessageHistory } from '@langchain/community/stores/message/upstash_redis'
 import { mapStoredMessageToChatMessage, AIMessage, HumanMessage, StoredMessage, BaseMessage } from '@langchain/core/messages'
@@ -6,6 +7,24 @@ import { FlowiseMemory, IMessage, INode, INodeData, INodeParams, MemoryMethods,
 import { convertBaseMessagetoIMessage, getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 import { ICommonObject } from '../../../src/Interface'
 
+let redisClientSingleton: Redis
+let redisClientOption: RedisConfigNodejs
+
+const getRedisClientbyOption = (option: RedisConfigNodejs) => {
+    if (!redisClientSingleton) {
+        // if client doesn't exists
+        redisClientSingleton = new Redis(option)
+        redisClientOption = option
+        return redisClientSingleton
+    } else if (redisClientSingleton && !isEqual(option, redisClientOption)) {
+        // if client exists but option changed
+        redisClientSingleton = new Redis(option)
+        redisClientOption = option
+        return redisClientSingleton
+    }
+    return redisClientSingleton
+}
+
 class UpstashRedisBackedChatMemory_Memory implements INode {
     label: string
     name: string
@@ -75,7 +94,7 @@ const initalizeUpstashRedis = async (nodeData: INodeData, options: ICommonObject
     const credentialData = await getCredentialData(nodeData.credential ?? '', options)
     const upstashRestToken = getCredentialParam('upstashRestToken', credentialData, nodeData)
 
-    const client = new Redis({
+    const client = getRedisClientbyOption({
         url: baseURL,
         token: upstashRestToken
     })

packages/components/nodes/memory/ZepMemory/ZepMemory.ts

@@ -17,7 +17,7 @@ class ZepMemory_Memory implements INode {
     inputs: INodeParams[]
 
     constructor() {
-        this.label = 'Zep Memory'
+        this.label = 'Zep Memory - Open Source'
         this.name = 'ZepMemory'
         this.version = 2.0
         this.type = 'ZepMemory'
@@ -97,11 +97,11 @@ class ZepMemory_Memory implements INode {
     }
 
     async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
-        return await initalizeZep(nodeData, options)
+        return await initializeZep(nodeData, options)
     }
 }
 
-const initalizeZep = async (nodeData: INodeData, options: ICommonObject): Promise<ZepMemory> => {
+const initializeZep = async (nodeData: INodeData, options: ICommonObject): Promise<ZepMemory> => {
     const baseURL = nodeData.inputs?.baseURL as string
     const aiPrefix = nodeData.inputs?.aiPrefix as string
     const humanPrefix = nodeData.inputs?.humanPrefix as string

packages/components/nodes/memory/ZepMemoryCloud/ZepMemoryCloud.ts

@@ -0,0 +1,181 @@
+import { IMessage, INode, INodeData, INodeParams, MemoryMethods, MessageType } from '../../../src/Interface'
+import { convertBaseMessagetoIMessage, getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { ZepMemory, ZepMemoryInput } from '@getzep/zep-cloud/langchain'
+
+import { ICommonObject } from '../../../src'
+import { InputValues, MemoryVariables, OutputValues } from 'langchain/memory'
+import { BaseMessage } from 'langchain/schema'
+
+class ZepMemoryCloud_Memory implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    credential: INodeParams
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'Zep Memory - Cloud'
+        this.name = 'ZepMemoryCloud'
+        this.version = 2.0
+        this.type = 'ZepMemory'
+        this.icon = 'zep.svg'
+        this.category = 'Memory'
+        this.description = 'Summarizes the conversation and stores the memory in zep server'
+        this.baseClasses = [this.type, ...getBaseClasses(ZepMemory)]
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            optional: true,
+            description: 'Configure JWT authentication on your Zep instance (Optional)',
+            credentialNames: ['zepMemoryApi']
+        }
+        this.inputs = [
+            {
+                label: 'Session Id',
+                name: 'sessionId',
+                type: 'string',
+                description:
+                    'If not specified, a random id will be used. Learn <a target="_blank" href="https://docs.flowiseai.com/memory/long-term-memory#ui-and-embedded-chat">more</a>',
+                default: '',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Memory Type',
+                name: 'memoryType',
+                type: 'string',
+                default: 'perpetual',
+                description: 'Zep Memory Type, can be perpetual or message_window',
+                additionalParams: true
+            },
+            {
+                label: 'AI Prefix',
+                name: 'aiPrefix',
+                type: 'string',
+                default: 'ai',
+                additionalParams: true
+            },
+            {
+                label: 'Human Prefix',
+                name: 'humanPrefix',
+                type: 'string',
+                default: 'human',
+                additionalParams: true
+            },
+            {
+                label: 'Memory Key',
+                name: 'memoryKey',
+                type: 'string',
+                default: 'chat_history',
+                additionalParams: true
+            },
+            {
+                label: 'Input Key',
+                name: 'inputKey',
+                type: 'string',
+                default: 'input',
+                additionalParams: true
+            },
+            {
+                label: 'Output Key',
+                name: 'outputKey',
+                type: 'string',
+                default: 'text',
+                additionalParams: true
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        return await initializeZep(nodeData, options)
+    }
+}
+
+const initializeZep = async (nodeData: INodeData, options: ICommonObject): Promise<ZepMemory> => {
+    const aiPrefix = nodeData.inputs?.aiPrefix as string
+    const humanPrefix = nodeData.inputs?.humanPrefix as string
+    const memoryKey = nodeData.inputs?.memoryKey as string
+    const inputKey = nodeData.inputs?.inputKey as string
+
+    const memoryType = nodeData.inputs?.memoryType as 'perpetual' | 'message_window'
+    const sessionId = nodeData.inputs?.sessionId as string
+
+    const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+    const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
+    const obj: ZepMemoryInput & ZepMemoryExtendedInput = {
+        apiKey,
+        aiPrefix,
+        humanPrefix,
+        memoryKey,
+        sessionId,
+        inputKey,
+        memoryType: memoryType,
+        returnMessages: true
+    }
+
+    return new ZepMemoryExtended(obj)
+}
+
+interface ZepMemoryExtendedInput {
+    memoryType?: 'perpetual' | 'message_window'
+}
+
+class ZepMemoryExtended extends ZepMemory implements MemoryMethods {
+    memoryType: 'perpetual' | 'message_window'
+
+    constructor(fields: ZepMemoryInput & ZepMemoryExtendedInput) {
+        super(fields)
+        this.memoryType = fields.memoryType ?? 'perpetual'
+    }
+
+    async loadMemoryVariables(values: InputValues, overrideSessionId = ''): Promise<MemoryVariables> {
+        if (overrideSessionId) {
+            this.sessionId = overrideSessionId
+        }
+        return super.loadMemoryVariables({ ...values, memoryType: this.memoryType })
+    }
+
+    async saveContext(inputValues: InputValues, outputValues: OutputValues, overrideSessionId = ''): Promise<void> {
+        if (overrideSessionId) {
+            this.sessionId = overrideSessionId
+        }
+        return super.saveContext(inputValues, outputValues)
+    }
+
+    async clear(overrideSessionId = ''): Promise<void> {
+        if (overrideSessionId) {
+            this.sessionId = overrideSessionId
+        }
+        return super.clear()
+    }
+
+    async getChatMessages(overrideSessionId = '', returnBaseMessages = false): Promise<IMessage[] | BaseMessage[]> {
+        const id = overrideSessionId ? overrideSessionId : this.sessionId
+        const memoryVariables = await this.loadMemoryVariables({}, id)
+        const baseMessages = memoryVariables[this.memoryKey]
+        return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages)
+    }
+
+    async addChatMessages(msgArray: { text: string; type: MessageType }[], overrideSessionId = ''): Promise<void> {
+        const id = overrideSessionId ? overrideSessionId : this.sessionId
+        const input = msgArray.find((msg) => msg.type === 'userMessage')
+        const output = msgArray.find((msg) => msg.type === 'apiMessage')
+        const inputValues = { [this.inputKey ?? 'input']: input?.text }
+        const outputValues = { output: output?.text }
+
+        await this.saveContext(inputValues, outputValues, id)
+    }
+
+    async clearChatMessages(overrideSessionId = ''): Promise<void> {
+        const id = overrideSessionId ? overrideSessionId : this.sessionId
+        await this.clear(id)
+    }
+}
+
+module.exports = { nodeClass: ZepMemoryCloud_Memory }

packages/components/nodes/memory/ZepMemoryCloud/zep.svg

@@ -0,0 +1,19 @@
+<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
+<circle cx="16" cy="16" r="14" fill="url(#paint0_linear_119_15736)"/>
+<path d="M12.6665 9.33333V12.6667H19.3332V9.33333C19.3332 8.59695 18.7362 8 17.9998 8H13.9998C13.2635 8 12.6665 8.59695 12.6665 9.33333Z" fill="white" stroke="white" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M19.3333 14.6667H12.6667C12.2985 14.6667 12 14.9652 12 15.3334V18.0001C12 18.3683 12.2985 18.6667 12.6667 18.6667H19.3333C19.7015 18.6667 20 18.3683 20 18.0001V15.3334C20 14.9652 19.7015 14.6667 19.3333 14.6667Z" fill="white" stroke="white" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M10 14.6667V20.0001" stroke="white" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M22 14.6667V20.0001" stroke="white" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M14 20.6667V24.0001" stroke="white" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M18 20.6667V24.0001" stroke="white" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M13.3335 24.6667H14.6668" stroke="white" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M17.3335 24.6667H18.6668" stroke="white" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
+<circle cx="14.3889" cy="10.0556" r="0.888889" fill="#7734A6"/>
+<circle cx="17.7224" cy="10.0556" r="0.888889" fill="#7734A6"/>
+<defs>
+<linearGradient id="paint0_linear_119_15736" x1="5.5" y1="6.5" x2="24.5" y2="26" gradientUnits="userSpaceOnUse">
+<stop stop-color="#6A31A6"/>
+<stop offset="1" stop-color="#9A3BA3"/>
+</linearGradient>
+</defs>
+</svg>

packages/components/nodes/speechtotext/assemblyai/AssemblyAI.ts

@@ -0,0 +1,33 @@
+import { INode, INodeParams } from '../../../src/Interface'
+
+class AssemblyAI_SpeechToText implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs?: INodeParams[]
+    credential: INodeParams
+
+    constructor() {
+        this.label = 'AssemblyAI'
+        this.name = 'assemblyAI'
+        this.version = 1.0
+        this.type = 'AssemblyAI'
+        this.icon = 'assemblyai.png'
+        this.category = 'SpeechToText'
+        this.baseClasses = [this.type]
+        this.inputs = []
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['assemblyAIApi']
+        }
+    }
+}
+
+module.exports = { nodeClass: AssemblyAI_SpeechToText }

packages/components/nodes/vectorstores/Zep/Zep.ts

@@ -22,7 +22,7 @@ class Zep_VectorStores implements INode {
     outputs: INodeOutputsValue[]
 
     constructor() {
-        this.label = 'Zep'
+        this.label = 'Zep Collection - Open Source'
         this.name = 'zep'
         this.version = 2.0
         this.type = 'Zep'

packages/components/nodes/vectorstores/Zep/Zep_Existing.ts

@@ -20,7 +20,7 @@ class Zep_Existing_VectorStores implements INode {
     outputs: INodeOutputsValue[]
 
     constructor() {
-        this.label = 'Zep Load Existing Index'
+        this.label = 'Zep Load Existing Index - Open Source'
         this.name = 'zepExistingIndex'
         this.version = 1.0
         this.type = 'Zep'

packages/components/nodes/vectorstores/Zep/Zep_Upsert.ts

@@ -20,7 +20,7 @@ class Zep_Upsert_VectorStores implements INode {
     outputs: INodeOutputsValue[]
 
     constructor() {
-        this.label = 'Zep Upsert Document'
+        this.label = 'Zep Upsert Document - Open Source'
         this.name = 'zepUpsert'
         this.version = 1.0
         this.type = 'Zep'

packages/components/nodes/vectorstores/ZepCloud/ZepCloud.ts

@@ -0,0 +1,231 @@
+import { flatten } from 'lodash'
+import { IDocument, ZepClient } from '@getzep/zep-cloud'
+import { IZepConfig, ZepVectorStore } from '@getzep/zep-cloud/langchain'
+import { Embeddings } from 'langchain/embeddings/base'
+import { Document } from 'langchain/document'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { addMMRInputParams, resolveVectorStoreOrRetriever } from '../VectorStoreUtils'
+import { FakeEmbeddings } from 'langchain/embeddings/fake'
+
+class Zep_CloudVectorStores implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    badge: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    credential: INodeParams
+    outputs: INodeOutputsValue[]
+
+    constructor() {
+        this.label = 'Zep Collection - Cloud'
+        this.name = 'zepCloud'
+        this.version = 2.0
+        this.type = 'Zep'
+        this.icon = 'zep.svg'
+        this.category = 'Vector Stores'
+        this.description =
+            'Upsert embedded data and perform similarity or mmr search upon query using Zep, a fast and scalable building block for LLM apps'
+        this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
+        this.badge = 'NEW'
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            optional: false,
+            description: 'Configure JWT authentication on your Zep instance (Optional)',
+            credentialNames: ['zepMemoryApi']
+        }
+        this.inputs = [
+            {
+                label: 'Document',
+                name: 'document',
+                type: 'Document',
+                list: true,
+                optional: true
+            },
+            {
+                label: 'Zep Collection',
+                name: 'zepCollection',
+                type: 'string',
+                placeholder: 'my-first-collection'
+            },
+            {
+                label: 'Zep Metadata Filter',
+                name: 'zepMetadataFilter',
+                type: 'json',
+                optional: true,
+                additionalParams: true
+            },
+            {
+                label: 'Top K',
+                name: 'topK',
+                description: 'Number of top results to fetch. Default to 4',
+                placeholder: '4',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            }
+        ]
+        addMMRInputParams(this.inputs)
+        this.outputs = [
+            {
+                label: 'Zep Retriever',
+                name: 'retriever',
+                baseClasses: this.baseClasses
+            },
+            {
+                label: 'Zep Vector Store',
+                name: 'vectorStore',
+                baseClasses: [this.type, ...getBaseClasses(ZepVectorStore)]
+            }
+        ]
+    }
+
+    //@ts-ignore
+    vectorStoreMethods = {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<void> {
+            const zepCollection = nodeData.inputs?.zepCollection as string
+            const docs = nodeData.inputs?.document as Document[]
+            const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+            const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
+            const flattenDocs = docs && docs.length ? flatten(docs) : []
+            const finalDocs = []
+            for (let i = 0; i < flattenDocs.length; i += 1) {
+                if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                    finalDocs.push(new Document(flattenDocs[i]))
+                }
+            }
+            const client = await ZepClient.init(apiKey)
+            const zepConfig = {
+                apiKey: apiKey,
+                collectionName: zepCollection,
+                client
+            }
+            try {
+                await ZepVectorStore.fromDocuments(finalDocs, new FakeEmbeddings(), zepConfig)
+            } catch (e) {
+                throw new Error(e)
+            }
+        }
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const zepCollection = nodeData.inputs?.zepCollection as string
+        const zepMetadataFilter = nodeData.inputs?.zepMetadataFilter
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
+
+        const zepConfig: IZepConfig & Partial<ZepFilter> = {
+            apiKey,
+            collectionName: zepCollection
+        }
+        if (zepMetadataFilter) {
+            zepConfig.filter = typeof zepMetadataFilter === 'object' ? zepMetadataFilter : JSON.parse(zepMetadataFilter)
+        }
+        zepConfig.client = await ZepClient.init(zepConfig.apiKey)
+        const vectorStore = await ZepExistingVS.init(zepConfig)
+        return resolveVectorStoreOrRetriever(nodeData, vectorStore)
+    }
+}
+
+interface ZepFilter {
+    filter: Record<string, any>
+}
+
+function zepDocsToDocumentsAndScore(results: IDocument[]): [Document, number][] {
+    return results.map((d) => [
+        new Document({
+            pageContent: d.content,
+            metadata: d.metadata
+        }),
+        d.score ? d.score : 0
+    ])
+}
+
+function assignMetadata(value: string | Record<string, unknown> | object | undefined): Record<string, unknown> | undefined {
+    if (typeof value === 'object' && value !== null) {
+        return value as Record<string, unknown>
+    }
+    if (value !== undefined) {
+        console.warn('Metadata filters must be an object, Record, or undefined.')
+    }
+    return undefined
+}
+
+class ZepExistingVS extends ZepVectorStore {
+    filter?: Record<string, any>
+    args?: IZepConfig & Partial<ZepFilter>
+
+    constructor(embeddings: Embeddings, args: IZepConfig & Partial<ZepFilter>) {
+        super(embeddings, args)
+        this.filter = args.filter
+        this.args = args
+    }
+
+    async initializeCollection(args: IZepConfig & Partial<ZepFilter>) {
+        this.client = await ZepClient.init(args.apiKey, args.apiUrl)
+        try {
+            this.collection = await this.client.document.getCollection(args.collectionName)
+        } catch (err) {
+            if (err instanceof Error) {
+                if (err.name === 'NotFoundError') {
+                    await this.createNewCollection(args)
+                } else {
+                    throw err
+                }
+            }
+        }
+    }
+
+    async createNewCollection(args: IZepConfig & Partial<ZepFilter>) {
+        this.collection = await this.client.document.addCollection({
+            name: args.collectionName,
+            description: args.description,
+            metadata: args.metadata
+        })
+    }
+
+    async similaritySearchVectorWithScore(
+        query: number[],
+        k: number,
+        filter?: Record<string, unknown> | undefined
+    ): Promise<[Document, number][]> {
+        if (filter && this.filter) {
+            throw new Error('cannot provide both `filter` and `this.filter`')
+        }
+        const _filters = filter ?? this.filter
+        const ANDFilters = []
+        for (const filterKey in _filters) {
+            let filterVal = _filters[filterKey]
+            if (typeof filterVal === 'string') filterVal = `"${filterVal}"`
+            ANDFilters.push({ jsonpath: `$[*] ? (@.${filterKey} == ${filterVal})` })
+        }
+        const newfilter = {
+            where: { and: ANDFilters }
+        }
+        await this.initializeCollection(this.args!).catch((err) => {
+            console.error('Error initializing collection:', err)
+            throw err
+        })
+        const results = await this.collection.search(
+            {
+                embedding: new Float32Array(query),
+                metadata: assignMetadata(newfilter)
+            },
+            k
+        )
+        return zepDocsToDocumentsAndScore(results)
+    }
+
+    static async fromExistingIndex(embeddings: Embeddings, dbConfig: IZepConfig & Partial<ZepFilter>): Promise<ZepVectorStore> {
+        return new this(embeddings, dbConfig)
+    }
+}
+
+module.exports = { nodeClass: Zep_CloudVectorStores }

packages/components/nodes/vectorstores/ZepCloud/zep.svg

@@ -0,0 +1,19 @@
+<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
+<circle cx="16" cy="16" r="14" fill="url(#paint0_linear_119_15736)"/>
+<path d="M12.6665 9.33333V12.6667H19.3332V9.33333C19.3332 8.59695 18.7362 8 17.9998 8H13.9998C13.2635 8 12.6665 8.59695 12.6665 9.33333Z" fill="white" stroke="white" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M19.3333 14.6667H12.6667C12.2985 14.6667 12 14.9652 12 15.3334V18.0001C12 18.3683 12.2985 18.6667 12.6667 18.6667H19.3333C19.7015 18.6667 20 18.3683 20 18.0001V15.3334C20 14.9652 19.7015 14.6667 19.3333 14.6667Z" fill="white" stroke="white" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M10 14.6667V20.0001" stroke="white" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M22 14.6667V20.0001" stroke="white" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M14 20.6667V24.0001" stroke="white" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M18 20.6667V24.0001" stroke="white" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M13.3335 24.6667H14.6668" stroke="white" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M17.3335 24.6667H18.6668" stroke="white" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
+<circle cx="14.3889" cy="10.0556" r="0.888889" fill="#7734A6"/>
+<circle cx="17.7224" cy="10.0556" r="0.888889" fill="#7734A6"/>
+<defs>
+<linearGradient id="paint0_linear_119_15736" x1="5.5" y1="6.5" x2="24.5" y2="26" gradientUnits="userSpaceOnUse">
+<stop stop-color="#6A31A6"/>
+<stop offset="1" stop-color="#9A3BA3"/>
+</linearGradient>
+</defs>
+</svg>

packages/components/package.json

@@ -1,6 +1,6 @@
 {
     "name": "flowise-components",
-    "version": "1.6.0",
+    "version": "1.6.1",
     "description": "Flowiseai Components",
     "main": "dist/src/index",
     "types": "dist/src/index.d.ts",
@@ -22,6 +22,7 @@
         "@datastax/astra-db-ts": "^0.1.2",
         "@dqbd/tiktoken": "^1.0.7",
         "@elastic/elasticsearch": "^8.9.0",
+        "@getzep/zep-cloud": "npm:@getzep/zep-js@next",
         "@getzep/zep-js": "^0.9.0",
         "@gomomento/sdk": "^1.51.1",
         "@gomomento/sdk-core": "^1.51.1",
@@ -45,6 +46,7 @@
         "@upstash/redis": "1.22.1",
         "@zilliz/milvus2-sdk-node": "^2.2.24",
         "apify-client": "^2.7.1",
+        "assemblyai": "^4.2.2",
         "axios": "1.6.2",
         "cheerio": "^1.0.0-rc.12",
         "chromadb": "^1.5.11",
@@ -62,9 +64,9 @@
         "ioredis": "^5.3.2",
         "jsonpointer": "^5.0.1",
         "langchain": "^0.1.20",
-        "langfuse": "2.0.2",
-        "langfuse-langchain": "^2.6.2-alpha.0",
-        "langsmith": "0.0.63",
+        "langfuse": "3.1.0",
+        "langfuse-langchain": "^3.1.0",
+        "langsmith": "0.1.6",
         "linkifyjs": "^4.1.1",
         "llamaindex": "^0.0.48",
         "lunary": "^0.6.16",

packages/components/src/Interface.ts

@@ -21,6 +21,8 @@ export type CommonType = string | number | boolean | undefined | null
 
 export type MessageType = 'apiMessage' | 'userMessage'
 
+export type ImageDetail = 'auto' | 'low' | 'high'
+
 /**
  * Others
  */
@@ -146,6 +148,33 @@ export interface IUsedTool {
     toolOutput: string | object
 }
 
+export interface IFileUpload {
+    data?: string
+    type: string
+    name: string
+    mime: string
+}
+
+export interface IMultiModalOption {
+    image?: Record<string, any>
+    audio?: Record<string, any>
+}
+
+export type MessageContentText = {
+    type: 'text'
+    text: string
+}
+
+export type MessageContentImageUrl = {
+    type: 'image_url'
+    image_url:
+        | string
+        | {
+              url: string
+              detail?: ImageDetail
+          }
+}
+
 /**
  * Classes
  */

packages/components/src/agents.ts

@@ -3,12 +3,23 @@ import { ChainValues } from '@langchain/core/utils/types'
 import { AgentStep, AgentAction } from '@langchain/core/agents'
 import { BaseMessage, FunctionMessage, AIMessage } from '@langchain/core/messages'
 import { OutputParserException } from '@langchain/core/output_parsers'
+import { BaseLanguageModel } from '@langchain/core/language_models/base'
 import { CallbackManager, CallbackManagerForChainRun, Callbacks } from '@langchain/core/callbacks/manager'
-import { ToolInputParsingException, Tool } from '@langchain/core/tools'
-import { Runnable } from '@langchain/core/runnables'
+import { ToolInputParsingException, Tool, StructuredToolInterface } from '@langchain/core/tools'
+import { Runnable, RunnableSequence, RunnablePassthrough } from '@langchain/core/runnables'
 import { Serializable } from '@langchain/core/load/serializable'
+import { renderTemplate } from '@langchain/core/prompts'
 import { BaseChain, SerializedLLMChain } from 'langchain/chains'
-import { AgentExecutorInput, BaseSingleActionAgent, BaseMultiActionAgent, RunnableAgent, StoppingMethod } from 'langchain/agents'
+import {
+    CreateReactAgentParams,
+    AgentExecutorInput,
+    AgentActionOutputParser,
+    BaseSingleActionAgent,
+    BaseMultiActionAgent,
+    RunnableAgent,
+    StoppingMethod
+} from 'langchain/agents'
+import { formatLogToString } from 'langchain/agents/format_scratchpad/log'
 
 export const SOURCE_DOCUMENTS_PREFIX = '\n\n----FLOWISE_SOURCE_DOCUMENTS----\n\n'
 type AgentFinish = {
@@ -647,3 +658,110 @@ export const formatAgentSteps = (steps: AgentStep[]): BaseMessage[] =>
             return [new AIMessage(action.log)]
         }
     })
+
+const renderTextDescription = (tools: StructuredToolInterface[]): string => {
+    return tools.map((tool) => `${tool.name}: ${tool.description}`).join('\n')
+}
+
+export const createReactAgent = async ({ llm, tools, prompt }: CreateReactAgentParams) => {
+    const missingVariables = ['tools', 'tool_names', 'agent_scratchpad'].filter((v) => !prompt.inputVariables.includes(v))
+    if (missingVariables.length > 0) {
+        throw new Error(`Provided prompt is missing required input variables: ${JSON.stringify(missingVariables)}`)
+    }
+    const toolNames = tools.map((tool) => tool.name)
+    const partialedPrompt = await prompt.partial({
+        tools: renderTextDescription(tools),
+        tool_names: toolNames.join(', ')
+    })
+    // TODO: Add .bind to core runnable interface.
+    const llmWithStop = (llm as BaseLanguageModel).bind({
+        stop: ['\nObservation:']
+    })
+    const agent = RunnableSequence.from([
+        RunnablePassthrough.assign({
+            //@ts-ignore
+            agent_scratchpad: (input: { steps: AgentStep[] }) => formatLogToString(input.steps)
+        }),
+        partialedPrompt,
+        llmWithStop,
+        new ReActSingleInputOutputParser({
+            toolNames
+        })
+    ])
+    return agent
+}
+
+class ReActSingleInputOutputParser extends AgentActionOutputParser {
+    lc_namespace = ['langchain', 'agents', 'react']
+
+    private toolNames: string[]
+    private FINAL_ANSWER_ACTION = 'Final Answer:'
+    private FINAL_ANSWER_AND_PARSABLE_ACTION_ERROR_MESSAGE = 'Parsing LLM output produced both a final answer and a parse-able action:'
+    private FORMAT_INSTRUCTIONS = `Use the following format:
+
+Question: the input question you must answer
+Thought: you should always think about what to do
+Action: the action to take, should be one of [{tool_names}]
+Action Input: the input to the action
+Observation: the result of the action
+... (this Thought/Action/Action Input/Observation can repeat N times)
+Thought: I now know the final answer
+Final Answer: the final answer to the original input question`
+
+    constructor(fields: { toolNames: string[] }) {
+        super(...arguments)
+        this.toolNames = fields.toolNames
+    }
+
+    /**
+     * Parses the given text into an AgentAction or AgentFinish object. If an
+     * output fixing parser is defined, uses it to parse the text.
+     * @param text Text to parse.
+     * @returns Promise that resolves to an AgentAction or AgentFinish object.
+     */
+    async parse(text: string): Promise<AgentAction | AgentFinish> {
+        const includesAnswer = text.includes(this.FINAL_ANSWER_ACTION)
+        const regex = /Action\s*\d*\s*:[\s]*(.*?)[\s]*Action\s*\d*\s*Input\s*\d*\s*:[\s]*(.*)/
+        const actionMatch = text.match(regex)
+        if (actionMatch) {
+            if (includesAnswer) {
+                throw new Error(`${this.FINAL_ANSWER_AND_PARSABLE_ACTION_ERROR_MESSAGE}: ${text}`)
+            }
+
+            const action = actionMatch[1]
+            const actionInput = actionMatch[2]
+            const toolInput = actionInput.trim().replace(/"/g, '')
+
+            return {
+                tool: action,
+                toolInput,
+                log: text
+            }
+        }
+
+        if (includesAnswer) {
+            const finalAnswerText = text.split(this.FINAL_ANSWER_ACTION)[1].trim()
+            return {
+                returnValues: {
+                    output: finalAnswerText
+                },
+                log: text
+            }
+        }
+
+        // Instead of throwing Error, we return a AgentFinish object
+        return { returnValues: { output: text }, log: text }
+    }
+
+    /**
+     * Returns the format instructions as a string. If the 'raw' option is
+     * true, returns the raw FORMAT_INSTRUCTIONS.
+     * @param options Options for getting the format instructions.
+     * @returns Format instructions as a string.
+     */
+    getFormatInstructions(): string {
+        return renderTemplate(this.FORMAT_INSTRUCTIONS, 'f-string', {
+            tool_names: this.toolNames.join(', ')
+        })
+    }
+}

packages/components/src/index.ts

@@ -6,3 +6,4 @@ dotenv.config({ path: envPath, override: true })
 
 export * from './Interface'
 export * from './utils'
+export * from './speechToText'

packages/components/src/multiModalUtils.ts

@@ -0,0 +1,48 @@
+import { ICommonObject, IFileUpload, IMultiModalOption, INodeData, MessageContentImageUrl } from './Interface'
+import { ChatOpenAI as LangchainChatOpenAI } from 'langchain/chat_models/openai'
+import path from 'path'
+import { getStoragePath } from './utils'
+import fs from 'fs'
+
+export const addImagesToMessages = (
+    nodeData: INodeData,
+    options: ICommonObject,
+    multiModalOption?: IMultiModalOption
+): MessageContentImageUrl[] => {
+    const imageContent: MessageContentImageUrl[] = []
+    let model = nodeData.inputs?.model
+
+    if (model instanceof LangchainChatOpenAI && multiModalOption) {
+        // Image Uploaded
+        if (multiModalOption.image && multiModalOption.image.allowImageUploads && options?.uploads && options?.uploads.length > 0) {
+            const imageUploads = getImageUploads(options.uploads)
+            for (const upload of imageUploads) {
+                let bf = upload.data
+                if (upload.type == 'stored-file') {
+                    const filePath = path.join(getStoragePath(), options.chatflowid, options.chatId, upload.name)
+
+                    // as the image is stored in the server, read the file and convert it to base64
+                    const contents = fs.readFileSync(filePath)
+                    bf = 'data:' + upload.mime + ';base64,' + contents.toString('base64')
+
+                    imageContent.push({
+                        type: 'image_url',
+                        image_url: {
+                            url: bf,
+                            detail: multiModalOption.image.imageResolution ?? 'low'
+                        }
+                    })
+                }
+            }
+        }
+    }
+    return imageContent
+}
+
+export const getAudioUploads = (uploads: IFileUpload[]) => {
+    return uploads.filter((upload: IFileUpload) => upload.mime.startsWith('audio/'))
+}
+
+export const getImageUploads = (uploads: IFileUpload[]) => {
+    return uploads.filter((upload: IFileUpload) => upload.mime.startsWith('image/'))
+}

packages/components/src/speechToText.ts

@@ -0,0 +1,51 @@
+import { ICommonObject, IFileUpload } from './Interface'
+import { getCredentialData, getStoragePath } from './utils'
+import { type ClientOptions, OpenAIClient } from '@langchain/openai'
+import fs from 'fs'
+import path from 'path'
+import { AssemblyAI } from 'assemblyai'
+
+export const convertSpeechToText = async (upload: IFileUpload, speechToTextConfig: ICommonObject, options: ICommonObject) => {
+    if (speechToTextConfig) {
+        const credentialId = speechToTextConfig.credentialId as string
+        const credentialData = await getCredentialData(credentialId ?? '', options)
+        const filePath = path.join(getStoragePath(), options.chatflowid, options.chatId, upload.name)
+
+        const audio_file = fs.createReadStream(filePath)
+
+        if (speechToTextConfig.name === 'openAIWhisper') {
+            const openAIClientOptions: ClientOptions = {
+                apiKey: credentialData.openAIApiKey
+            }
+            const openAIClient = new OpenAIClient(openAIClientOptions)
+
+            const transcription = await openAIClient.audio.transcriptions.create({
+                file: audio_file,
+                model: 'whisper-1',
+                language: speechToTextConfig?.language,
+                temperature: speechToTextConfig?.temperature ? parseFloat(speechToTextConfig.temperature) : undefined,
+                prompt: speechToTextConfig?.prompt
+            })
+            if (transcription?.text) {
+                return transcription.text
+            }
+        } else if (speechToTextConfig.name === 'assemblyAiTranscribe') {
+            const client = new AssemblyAI({
+                apiKey: credentialData.assemblyAIApiKey
+            })
+
+            const params = {
+                audio: audio_file,
+                speaker_labels: false
+            }
+
+            const transcription = await client.transcripts.transcribe(params)
+            if (transcription?.text) {
+                return transcription.text
+            }
+        }
+    } else {
+        throw new Error('Speech to text is not selected, but found a recorded audio file. Please fix the chain.')
+    }
+    return undefined
+}

packages/components/src/utils.ts

@@ -290,22 +290,12 @@ function getURLsFromHTML(htmlBody: string, baseURL: string): string[] {
     const linkElements = dom.window.document.querySelectorAll('a')
     const urls: string[] = []
     for (const linkElement of linkElements) {
-        if (linkElement.href.slice(0, 1) === '/') {
-            try {
-                const urlObj = new URL(baseURL + linkElement.href)
-                urls.push(urlObj.href) //relative
-            } catch (err) {
-                if (process.env.DEBUG === 'true') console.error(`error with relative url: ${err.message}`)
-                continue
-            }
-        } else {
-            try {
-                const urlObj = new URL(linkElement.href)
-                urls.push(urlObj.href) //absolute
-            } catch (err) {
-                if (process.env.DEBUG === 'true') console.error(`error with absolute url: ${err.message}`)
-                continue
-            }
+        try {
+            const urlObj = new URL(linkElement.href, baseURL)
+            urls.push(urlObj.href)
+        } catch (err) {
+            if (process.env.DEBUG === 'true') console.error(`error with scraped URL: ${err.message}`)
+            continue
         }
     }
     return urls
@@ -365,7 +355,7 @@ async function crawl(baseURL: string, currentURL: string, pages: string[], limit
         }
 
         const htmlBody = await resp.text()
-        const nextURLs = getURLsFromHTML(htmlBody, baseURL)
+        const nextURLs = getURLsFromHTML(htmlBody, currentURL)
         for (const nextURL of nextURLs) {
             pages = await crawl(baseURL, nextURL, pages, limit)
         }
@@ -780,3 +770,10 @@ export const prepareSandboxVars = (variables: IVariable[]) => {
     }
     return vars
 }
+
+/**
+ * Prepare storage path
+ */
+export const getStoragePath = (): string => {
+    return process.env.BLOB_STORAGE_PATH ? path.join(process.env.BLOB_STORAGE_PATH) : path.join(getUserHome(), '.flowise', 'storage')
+}

packages/server/.env.example

@@ -5,6 +5,7 @@ PORT=3000
 # APIKEY_PATH=/your_api_key_path/.flowise
 # SECRETKEY_PATH=/your_api_key_path/.flowise
 # LOG_PATH=/your_log_path/.flowise/logs
+# BLOB_STORAGE_PATH=/your_database_path/.flowise/storage
 
 # NUMBER_OF_PROXIES= 1
 

packages/server/marketplaces/chatflows/API Agent OpenAI.json

@@ -90,7 +90,7 @@
             "data": {
                 "id": "chatOpenAI_1",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"],
@@ -237,6 +237,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_1-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_1-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_1-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -257,7 +290,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {
@@ -437,7 +472,7 @@
             "data": {
                 "id": "chatOpenAI_2",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"],
@@ -584,6 +619,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_2-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_2-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_2-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -604,7 +672,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/API Agent.json

@@ -398,7 +398,7 @@
             "data": {
                 "id": "chatOpenAI_2",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"],
@@ -545,6 +545,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_2-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_2-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_2-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -565,7 +598,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {
@@ -597,7 +632,7 @@
             "data": {
                 "id": "chatOpenAI_1",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"],
@@ -744,6 +779,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_1-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_1-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_1-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -764,7 +832,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {
@@ -796,7 +866,7 @@
             "data": {
                 "id": "chatOpenAI_3",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"],
@@ -943,6 +1013,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_3-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_3-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_3-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -963,7 +1066,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/Advanced Structured Output Parser.json

@@ -181,7 +181,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -334,6 +334,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -355,7 +388,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/Antonym.json

@@ -177,7 +177,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"],
@@ -324,6 +324,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -344,7 +377,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/AutoGPT.json

@@ -253,7 +253,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"],
@@ -400,6 +400,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -420,7 +453,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/BabyAGI.json

@@ -346,7 +346,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -499,6 +499,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -520,7 +553,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/CSV Agent.json

@@ -72,7 +72,7 @@
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
                 "name": "chatOpenAI",
-                "version": 3,
+                "version": 5,
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"],
                 "category": "Chat Models",
@@ -218,6 +218,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -238,7 +271,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/Chat with a Podcast.json

@@ -196,7 +196,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -349,6 +349,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -370,7 +403,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/ChatGPTPlugin.json

@@ -217,7 +217,7 @@
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
                 "name": "chatOpenAI",
-                "version": 3,
+                "version": 5,
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"],
                 "category": "Chat Models",
@@ -363,6 +363,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -383,7 +416,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/Conversational Agent.json

@@ -158,7 +158,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"],
@@ -305,6 +305,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -325,7 +358,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/Conversational Retrieval Agent.json

@@ -489,7 +489,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -642,6 +642,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -663,7 +696,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/Conversational Retrieval QA Chain.json

@@ -371,7 +371,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -524,6 +524,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -545,7 +578,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/Flowise Docs QnA.json

@@ -378,7 +378,7 @@
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
                 "name": "chatOpenAI",
-                "version": 3,
+                "version": 5,
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"],
                 "category": "Chat Models",
@@ -524,6 +524,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -544,7 +577,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/IfElse.json

@@ -910,7 +910,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -1063,6 +1063,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -1084,7 +1117,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/Image Generation.json

@@ -454,7 +454,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -607,6 +607,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -628,7 +661,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/Input Moderation.json

@@ -166,7 +166,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -319,6 +319,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -340,7 +373,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/List Output Parser.json

@@ -225,7 +225,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -378,6 +378,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -399,7 +432,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/Long Term Memory.json

@@ -508,7 +508,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -661,6 +661,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -682,7 +715,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/Metadata Filter.json

@@ -455,7 +455,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -608,6 +608,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -629,7 +662,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/Multi Prompt Chain.json

@@ -280,7 +280,7 @@
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
                 "name": "chatOpenAI",
-                "version": 3,
+                "version": 5,
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"],
                 "category": "Chat Models",
@@ -426,6 +426,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -446,7 +479,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/Multi Retrieval QA Chain.json

@@ -390,7 +390,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -543,6 +543,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -564,7 +597,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/Multiple VectorDB.json

@@ -487,7 +487,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -640,6 +640,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -661,7 +694,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {
@@ -1013,7 +1048,7 @@
             "data": {
                 "id": "chatOpenAI_1",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -1166,6 +1201,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_1-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_1-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_1-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -1187,7 +1255,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {
@@ -1219,7 +1289,7 @@
             "data": {
                 "id": "chatOpenAI_2",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -1372,6 +1442,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_2-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_2-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_2-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -1393,7 +1496,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/OpenAI Agent.json

@@ -281,7 +281,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"],
@@ -428,6 +428,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -448,7 +481,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/Prompt Chaining with VectorStore.json

@@ -429,7 +429,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -582,6 +582,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -603,7 +636,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {
@@ -635,7 +670,7 @@
             "data": {
                 "id": "chatOpenAI_1",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -788,6 +823,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_1-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_1-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_1-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -809,7 +877,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/ReAct Agent.json

@@ -5,11 +5,11 @@
     "nodes": [
         {
             "width": 300,
-            "height": 143,
+            "height": 142,
             "id": "calculator_1",
             "position": {
-                "x": 664.1366474718458,
-                "y": 123.16419000640141
+                "x": 466.86432329033937,
+                "y": 230.0825123205457
             },
             "type": "customNode",
             "data": {
@@ -36,72 +36,177 @@
                 "selected": false
             },
             "positionAbsolute": {
-                "x": 664.1366474718458,
-                "y": 123.16419000640141
+                "x": 466.86432329033937,
+                "y": 230.0825123205457
             },
             "selected": false,
             "dragging": false
         },
         {
-            "width": 300,
-            "height": 277,
-            "id": "serper_0",
+            "id": "mrklAgentChat_0",
             "position": {
-                "x": 330.964079024626,
-                "y": 109.83185250619351
+                "x": 905.8535326018256,
+                "y": 388.58312223652564
             },
             "type": "customNode",
             "data": {
-                "id": "serper_0",
-                "label": "Serper",
-                "version": 1,
-                "name": "serper",
-                "type": "Serper",
-                "baseClasses": ["Serper", "Tool", "StructuredTool"],
-                "category": "Tools",
-                "description": "Wrapper around Serper.dev - Google Search API",
-                "inputParams": [
+                "id": "mrklAgentChat_0",
+                "label": "ReAct Agent for Chat Models",
+                "version": 3,
+                "name": "mrklAgentChat",
+                "type": "AgentExecutor",
+                "baseClasses": ["AgentExecutor", "BaseChain", "Runnable"],
+                "category": "Agents",
+                "description": "Agent that uses the ReAct logic to decide what action to take, optimized to be used with Chat Models",
+                "inputParams": [],
+                "inputAnchors": [
                     {
-                        "label": "Connect Credential",
-                        "name": "credential",
-                        "type": "credential",
-                        "credentialNames": ["serperApi"],
-                        "id": "serper_0-input-credential-credential"
+                        "label": "Allowed Tools",
+                        "name": "tools",
+                        "type": "Tool",
+                        "list": true,
+                        "id": "mrklAgentChat_0-input-tools-Tool"
+                    },
+                    {
+                        "label": "Chat Model",
+                        "name": "model",
+                        "type": "BaseChatModel",
+                        "id": "mrklAgentChat_0-input-model-BaseChatModel"
+                    },
+                    {
+                        "label": "Memory",
+                        "name": "memory",
+                        "type": "BaseChatMemory",
+                        "id": "mrklAgentChat_0-input-memory-BaseChatMemory"
                     }
                 ],
-                "inputAnchors": [],
-                "inputs": {},
+                "inputs": {
+                    "tools": ["{{calculator_1.data.instance}}", "{{serper_0.data.instance}}"],
+                    "model": "{{chatOpenAI_0.data.instance}}",
+                    "memory": "{{RedisBackedChatMemory_0.data.instance}}"
+                },
                 "outputAnchors": [
                     {
-                        "id": "serper_0-output-serper-Serper|Tool|StructuredTool",
-                        "name": "serper",
-                        "label": "Serper",
-                        "type": "Serper | Tool | StructuredTool"
+                        "id": "mrklAgentChat_0-output-mrklAgentChat-AgentExecutor|BaseChain|Runnable",
+                        "name": "mrklAgentChat",
+                        "label": "AgentExecutor",
+                        "description": "Agent that uses the ReAct logic to decide what action to take, optimized to be used with Chat Models",
+                        "type": "AgentExecutor | BaseChain | Runnable"
                     }
                 ],
                 "outputs": {},
                 "selected": false
             },
+            "width": 300,
+            "height": 330,
             "selected": false,
             "positionAbsolute": {
-                "x": 330.964079024626,
-                "y": 109.83185250619351
+                "x": 905.8535326018256,
+                "y": 388.58312223652564
             },
             "dragging": false
         },
         {
+            "id": "RedisBackedChatMemory_0",
+            "position": {
+                "x": 473.108799702029,
+                "y": 401.8098683245926
+            },
+            "type": "customNode",
+            "data": {
+                "id": "RedisBackedChatMemory_0",
+                "label": "Redis-Backed Chat Memory",
+                "version": 2,
+                "name": "RedisBackedChatMemory",
+                "type": "RedisBackedChatMemory",
+                "baseClasses": ["RedisBackedChatMemory", "BaseChatMemory", "BaseMemory"],
+                "category": "Memory",
+                "description": "Summarizes the conversation and stores the memory in Redis server",
+                "inputParams": [
+                    {
+                        "label": "Connect Credential",
+                        "name": "credential",
+                        "type": "credential",
+                        "optional": true,
+                        "credentialNames": ["redisCacheApi", "redisCacheUrlApi"],
+                        "id": "RedisBackedChatMemory_0-input-credential-credential"
+                    },
+                    {
+                        "label": "Session Id",
+                        "name": "sessionId",
+                        "type": "string",
+                        "description": "If not specified, a random id will be used. Learn <a target=\"_blank\" href=\"https://docs.flowiseai.com/memory/long-term-memory#ui-and-embedded-chat\">more</a>",
+                        "default": "",
+                        "additionalParams": true,
+                        "optional": true,
+                        "id": "RedisBackedChatMemory_0-input-sessionId-string"
+                    },
+                    {
+                        "label": "Session Timeouts",
+                        "name": "sessionTTL",
+                        "type": "number",
+                        "description": "Omit this parameter to make sessions never expire",
+                        "additionalParams": true,
+                        "optional": true,
+                        "id": "RedisBackedChatMemory_0-input-sessionTTL-number"
+                    },
+                    {
+                        "label": "Memory Key",
+                        "name": "memoryKey",
+                        "type": "string",
+                        "default": "chat_history",
+                        "additionalParams": true,
+                        "id": "RedisBackedChatMemory_0-input-memoryKey-string"
+                    },
+                    {
+                        "label": "Window Size",
+                        "name": "windowSize",
+                        "type": "number",
+                        "description": "Window of size k to surface the last k back-and-forth to use as memory.",
+                        "additionalParams": true,
+                        "optional": true,
+                        "id": "RedisBackedChatMemory_0-input-windowSize-number"
+                    }
+                ],
+                "inputAnchors": [],
+                "inputs": {
+                    "sessionId": "",
+                    "sessionTTL": "",
+                    "memoryKey": "chat_history",
+                    "windowSize": ""
+                },
+                "outputAnchors": [
+                    {
+                        "id": "RedisBackedChatMemory_0-output-RedisBackedChatMemory-RedisBackedChatMemory|BaseChatMemory|BaseMemory",
+                        "name": "RedisBackedChatMemory",
+                        "label": "RedisBackedChatMemory",
+                        "description": "Summarizes the conversation and stores the memory in Redis server",
+                        "type": "RedisBackedChatMemory | BaseChatMemory | BaseMemory"
+                    }
+                ],
+                "outputs": {},
+                "selected": false
+            },
             "width": 300,
-            "height": 574,
+            "height": 328,
+            "selected": false,
+            "positionAbsolute": {
+                "x": 473.108799702029,
+                "y": 401.8098683245926
+            },
+            "dragging": false
+        },
+        {
             "id": "chatOpenAI_0",
             "position": {
-                "x": -27.71074046118335,
-                "y": 243.62715178281059
+                "x": 81.2222202723384,
+                "y": 59.395597724017364
             },
             "type": "customNode",
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -254,6 +359,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -275,80 +413,78 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {
                         "id": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable",
                         "name": "chatOpenAI",
                         "label": "ChatOpenAI",
+                        "description": "Wrapper around OpenAI large language models that use the Chat endpoint",
                         "type": "ChatOpenAI | BaseChatModel | BaseLanguageModel | Runnable"
                     }
                 ],
                 "outputs": {},
                 "selected": false
             },
+            "width": 300,
+            "height": 573,
             "selected": false,
             "positionAbsolute": {
-                "x": -27.71074046118335,
-                "y": 243.62715178281059
+                "x": 81.2222202723384,
+                "y": 59.395597724017364
             },
             "dragging": false
         },
         {
-            "width": 300,
-            "height": 280,
-            "id": "mrklAgentChat_0",
+            "id": "serper_0",
             "position": {
-                "x": 1090.2058867451212,
-                "y": 423.2174695788541
+                "x": 466.4499611299051,
+                "y": -67.74721119468873
             },
             "type": "customNode",
             "data": {
-                "id": "mrklAgentChat_0",
-                "label": "ReAct Agent for Chat Models",
+                "id": "serper_0",
+                "label": "Serper",
                 "version": 1,
-                "name": "mrklAgentChat",
-                "type": "AgentExecutor",
-                "baseClasses": ["AgentExecutor", "BaseChain", "Runnable"],
-                "category": "Agents",
-                "description": "Agent that uses the ReAct logic to decide what action to take, optimized to be used with Chat Models",
-                "inputParams": [],
-                "inputAnchors": [
+                "name": "serper",
+                "type": "Serper",
+                "baseClasses": ["Serper", "Tool", "StructuredTool", "Runnable"],
+                "category": "Tools",
+                "description": "Wrapper around Serper.dev - Google Search API",
+                "inputParams": [
                     {
-                        "label": "Allowed Tools",
-                        "name": "tools",
-                        "type": "Tool",
-                        "list": true,
-                        "id": "mrklAgentChat_0-input-tools-Tool"
-                    },
-                    {
-                        "label": "Language Model",
-                        "name": "model",
-                        "type": "BaseLanguageModel",
-                        "id": "mrklAgentChat_0-input-model-BaseLanguageModel"
+                        "label": "Connect Credential",
+                        "name": "credential",
+                        "type": "credential",
+                        "credentialNames": ["serperApi"],
+                        "id": "serper_0-input-credential-credential"
                     }
                 ],
-                "inputs": {
-                    "tools": ["{{calculator_1.data.instance}}", "{{serper_0.data.instance}}"],
-                    "model": "{{chatOpenAI_0.data.instance}}"
-                },
+                "inputAnchors": [],
+                "inputs": {},
                 "outputAnchors": [
                     {
-                        "id": "mrklAgentChat_0-output-mrklAgentChat-AgentExecutor|BaseChain|Runnable",
-                        "name": "mrklAgentChat",
-                        "label": "AgentExecutor",
-                        "type": "AgentExecutor | BaseChain | Runnable"
+                        "id": "serper_0-output-serper-Serper|Tool|StructuredTool|Runnable",
+                        "name": "serper",
+                        "label": "Serper",
+                        "description": "Wrapper around Serper.dev - Google Search API",
+                        "type": "Serper | Tool | StructuredTool | Runnable"
                     }
                 ],
                 "outputs": {},
                 "selected": false
             },
+            "width": 300,
+            "height": 276,
+            "selected": false,
             "positionAbsolute": {
-                "x": 1090.2058867451212,
-                "y": 423.2174695788541
+                "x": 466.4499611299051,
+                "y": -67.74721119468873
             },
-            "selected": false
+            "dragging": false
         }
     ],
     "edges": [
@@ -358,32 +494,31 @@
             "target": "mrklAgentChat_0",
             "targetHandle": "mrklAgentChat_0-input-tools-Tool",
             "type": "buttonedge",
-            "id": "calculator_1-calculator_1-output-calculator-Calculator|Tool|StructuredTool|BaseLangChain-mrklAgentChat_0-mrklAgentChat_0-input-tools-Tool",
-            "data": {
-                "label": ""
-            }
+            "id": "calculator_1-calculator_1-output-calculator-Calculator|Tool|StructuredTool|BaseLangChain-mrklAgentChat_0-mrklAgentChat_0-input-tools-Tool"
         },
         {
-            "source": "serper_0",
-            "sourceHandle": "serper_0-output-serper-Serper|Tool|StructuredTool",
+            "source": "RedisBackedChatMemory_0",
+            "sourceHandle": "RedisBackedChatMemory_0-output-RedisBackedChatMemory-RedisBackedChatMemory|BaseChatMemory|BaseMemory",
             "target": "mrklAgentChat_0",
-            "targetHandle": "mrklAgentChat_0-input-tools-Tool",
+            "targetHandle": "mrklAgentChat_0-input-memory-BaseChatMemory",
             "type": "buttonedge",
-            "id": "serper_0-serper_0-output-serper-Serper|Tool|StructuredTool-mrklAgentChat_0-mrklAgentChat_0-input-tools-Tool",
-            "data": {
-                "label": ""
-            }
+            "id": "RedisBackedChatMemory_0-RedisBackedChatMemory_0-output-RedisBackedChatMemory-RedisBackedChatMemory|BaseChatMemory|BaseMemory-mrklAgentChat_0-mrklAgentChat_0-input-memory-BaseChatMemory"
         },
         {
             "source": "chatOpenAI_0",
             "sourceHandle": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable",
             "target": "mrklAgentChat_0",
-            "targetHandle": "mrklAgentChat_0-input-model-BaseLanguageModel",
+            "targetHandle": "mrklAgentChat_0-input-model-BaseChatModel",
             "type": "buttonedge",
-            "id": "chatOpenAI_0-chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable-mrklAgentChat_0-mrklAgentChat_0-input-model-BaseLanguageModel",
-            "data": {
-                "label": ""
-            }
+            "id": "chatOpenAI_0-chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable-mrklAgentChat_0-mrklAgentChat_0-input-model-BaseChatModel"
+        },
+        {
+            "source": "serper_0",
+            "sourceHandle": "serper_0-output-serper-Serper|Tool|StructuredTool|Runnable",
+            "target": "mrklAgentChat_0",
+            "targetHandle": "mrklAgentChat_0-input-tools-Tool",
+            "type": "buttonedge",
+            "id": "serper_0-serper_0-output-serper-Serper|Tool|StructuredTool|Runnable-mrklAgentChat_0-mrklAgentChat_0-input-tools-Tool"
         }
     ]
 }

packages/server/marketplaces/chatflows/SQL DB Chain.json

@@ -15,7 +15,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"],
@@ -162,6 +162,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -182,7 +215,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/SQL Prompt.json

@@ -175,7 +175,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -328,6 +328,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -349,7 +382,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {
@@ -381,7 +416,7 @@
             "data": {
                 "id": "chatOpenAI_1",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -534,6 +569,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_1-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_1-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_1-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -555,7 +623,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {
@@ -1313,7 +1383,7 @@
             "data": {
                 "id": "chatOpenAI_2",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -1466,6 +1536,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_2-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_2-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_2-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -1487,7 +1590,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/Simple Conversation Chain.json

@@ -16,7 +16,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -169,6 +169,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -190,7 +223,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/Structured Output Parser.json

@@ -16,7 +16,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -169,6 +169,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -190,7 +223,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/Translator.json

@@ -84,7 +84,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"],
@@ -231,6 +231,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -251,7 +284,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/WebBrowser.json

@@ -127,7 +127,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"],
@@ -274,6 +274,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -294,7 +327,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {
@@ -433,7 +468,7 @@
             "data": {
                 "id": "chatOpenAI_1",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"],
@@ -580,6 +615,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_1-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_1-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_1-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -600,7 +668,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/marketplaces/chatflows/WebPage QnA.json

@@ -394,7 +394,7 @@
             "data": {
                 "id": "chatOpenAI_0",
                 "label": "ChatOpenAI",
-                "version": 3,
+                "version": 5,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
                 "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
@@ -547,6 +547,39 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-baseOptions-json"
+                    },
+                    {
+                        "label": "Allow Image Uploads",
+                        "name": "allowImageUploads",
+                        "type": "boolean",
+                        "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent",
+                        "default": false,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-allowImageUploads-boolean"
+                    },
+                    {
+                        "label": "Image Resolution",
+                        "description": "This parameter controls the resolution in which the model views the image.",
+                        "name": "imageResolution",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "Low",
+                                "name": "low"
+                            },
+                            {
+                                "label": "High",
+                                "name": "high"
+                            },
+                            {
+                                "label": "Auto",
+                                "name": "auto"
+                            }
+                        ],
+                        "default": "low",
+                        "optional": false,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-imageResolution-options"
                     }
                 ],
                 "inputAnchors": [
@@ -568,7 +601,9 @@
                     "presencePenalty": "",
                     "timeout": "",
                     "basepath": "",
-                    "baseOptions": ""
+                    "baseOptions": "",
+                    "allowImageUploads": true,
+                    "imageResolution": "low"
                 },
                 "outputAnchors": [
                     {

packages/server/package.json

@@ -1,6 +1,6 @@
 {
     "name": "flowise",
-    "version": "1.5.0",
+    "version": "1.5.1",
     "description": "Flowiseai Server",
     "main": "dist/index",
     "types": "dist/index.d.ts",
@@ -48,6 +48,7 @@
         "@oclif/core": "^1.13.10",
         "async-mutex": "^0.4.0",
         "axios": "1.6.2",
+        "content-disposition": "0.5.4",
         "cors": "^2.8.5",
         "crypto-js": "^4.1.1",
         "dotenv": "^16.0.0",
@@ -70,6 +71,7 @@
         "winston": "^3.9.0"
     },
     "devDependencies": {
+        "@types/content-disposition": "0.5.8",
         "@types/cors": "^2.8.12",
         "@types/crypto-js": "^4.1.1",
         "@types/multer": "^1.4.7",

packages/server/src/Interface.ts

@@ -1,4 +1,4 @@
-import { ICommonObject, INode, INodeData as INodeDataFromComponent, INodeParams } from 'flowise-components'
+import { ICommonObject, IFileUpload, INode, INodeData as INodeDataFromComponent, INodeParams } from 'flowise-components'
 
 export type MessageType = 'apiMessage' | 'userMessage'
 
@@ -31,6 +31,7 @@ export interface IChatMessage {
     sourceDocuments?: string
     usedTools?: string
     fileAnnotations?: string
+    fileUploads?: string
     chatType: string
     chatId: string
     memoryType?: string
@@ -176,6 +177,7 @@ export interface IncomingInput {
     socketIOClientId?: string
     chatId?: string
     stopNodeId?: string
+    uploads?: IFileUpload[]
 }
 
 export interface IActiveChatflows {
@@ -212,3 +214,8 @@ export interface ICredentialReqBody {
 export interface ICredentialReturnResponse extends ICredential {
     plainDataObj: ICredentialDataDecrypted
 }
+
+export interface IUploadFileSizeAndTypes {
+    fileTypes: string[]
+    maxUploadSize: number
+}

packages/server/src/NodesPool.ts

@@ -4,6 +4,7 @@ import { Dirent } from 'fs'
 import { getNodeModulesPackagePath } from './utils'
 import { promises } from 'fs'
 import { ICommonObject } from 'flowise-components'
+import logger from './utils/logger'
 
 export class NodesPool {
     componentNodes: IComponentNodes = {}
@@ -28,36 +29,40 @@ export class NodesPool {
         return Promise.all(
             nodeFiles.map(async (file) => {
                 if (file.endsWith('.js')) {
-                    const nodeModule = await require(file)
+                    try {
+                        const nodeModule = await require(file)
 
-                    if (nodeModule.nodeClass) {
-                        const newNodeInstance = new nodeModule.nodeClass()
-                        newNodeInstance.filePath = file
+                        if (nodeModule.nodeClass) {
+                            const newNodeInstance = new nodeModule.nodeClass()
+                            newNodeInstance.filePath = file
 
-                        // Replace file icon with absolute path
-                        if (
-                            newNodeInstance.icon &&
-                            (newNodeInstance.icon.endsWith('.svg') ||
-                                newNodeInstance.icon.endsWith('.png') ||
-                                newNodeInstance.icon.endsWith('.jpg'))
-                        ) {
-                            const filePath = file.replace(/\\/g, '/').split('/')
-                            filePath.pop()
-                            const nodeIconAbsolutePath = `${filePath.join('/')}/${newNodeInstance.icon}`
-                            newNodeInstance.icon = nodeIconAbsolutePath
+                            // Replace file icon with absolute path
+                            if (
+                                newNodeInstance.icon &&
+                                (newNodeInstance.icon.endsWith('.svg') ||
+                                    newNodeInstance.icon.endsWith('.png') ||
+                                    newNodeInstance.icon.endsWith('.jpg'))
+                            ) {
+                                const filePath = file.replace(/\\/g, '/').split('/')
+                                filePath.pop()
+                                const nodeIconAbsolutePath = `${filePath.join('/')}/${newNodeInstance.icon}`
+                                newNodeInstance.icon = nodeIconAbsolutePath
 
-                            // Store icon path for componentCredentials
-                            if (newNodeInstance.credential) {
-                                for (const credName of newNodeInstance.credential.credentialNames) {
-                                    this.credentialIconPath[credName] = nodeIconAbsolutePath
+                                // Store icon path for componentCredentials
+                                if (newNodeInstance.credential) {
+                                    for (const credName of newNodeInstance.credential.credentialNames) {
+                                        this.credentialIconPath[credName] = nodeIconAbsolutePath
+                                    }
                                 }
                             }
-                        }
 
-                        const skipCategories = ['Analytic']
-                        if (!skipCategories.includes(newNodeInstance.category)) {
-                            this.componentNodes[newNodeInstance.name] = newNodeInstance
+                            const skipCategories = ['Analytic', 'SpeechToText']
+                            if (!skipCategories.includes(newNodeInstance.category)) {
+                                this.componentNodes[newNodeInstance.name] = newNodeInstance
+                            }
                         }
+                    } catch (err) {
+                        logger.error(`❌ [server]: Error during initDatabase with file ${file}:`, err)
                     }
                 }
             })

packages/server/src/commands/start.ts

@@ -23,6 +23,7 @@ export default class Start extends Command {
         CORS_ORIGINS: Flags.string(),
         IFRAME_ORIGINS: Flags.string(),
         DEBUG: Flags.string(),
+        BLOB_STORAGE_PATH: Flags.string(),
         APIKEY_PATH: Flags.string(),
         SECRETKEY_PATH: Flags.string(),
         FLOWISE_SECRETKEY_OVERWRITE: Flags.string(),
@@ -92,6 +93,9 @@ export default class Start extends Command {
         if (flags.FLOWISE_PASSWORD) process.env.FLOWISE_PASSWORD = flags.FLOWISE_PASSWORD
         if (flags.APIKEY_PATH) process.env.APIKEY_PATH = flags.APIKEY_PATH
 
+        // Storage
+        if (flags.BLOB_STORAGE_PATH) process.env.BLOB_STORAGE_PATH = flags.BLOB_STORAGE_PATH
+
         //API Configuration
         if (flags.FLOWISE_FILE_SIZE_LIMIT) process.env.FLOWISE_FILE_SIZE_LIMIT = flags.FLOWISE_FILE_SIZE_LIMIT
 

packages/server/src/database/entities/ChatFlow.ts

@@ -31,6 +31,9 @@ export class ChatFlow implements IChatFlow {
     @Column({ nullable: true, type: 'text' })
     analytic?: string
 
+    @Column({ nullable: true, type: 'text' })
+    speechToText?: string
+
     @CreateDateColumn()
     createdDate: Date
 

packages/server/src/database/entities/ChatMessage.ts

@@ -26,6 +26,9 @@ export class ChatMessage implements IChatMessage {
     @Column({ nullable: true, type: 'text' })
     fileAnnotations?: string
 
+    @Column({ nullable: true, type: 'text' })
+    fileUploads?: string
+
     @Column()
     chatType: string
 

packages/server/src/database/migrations/mysql/1701788586491-AddFileUploadsToChatMessage.ts

@@ -0,0 +1,12 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddFileUploadsToChatMessage1701788586491 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        const columnExists = await queryRunner.hasColumn('chat_message', 'fileUploads')
+        if (!columnExists) queryRunner.query(`ALTER TABLE \`chat_message\` ADD COLUMN \`fileUploads\` TEXT;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`chat_message\` DROP COLUMN \`fileUploads\`;`)
+    }
+}

packages/server/src/database/migrations/mysql/1706364937060-AddSpeechToText.ts

@@ -0,0 +1,12 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddSpeechToText1706364937060 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        const columnExists = await queryRunner.hasColumn('chat_flow', 'speechToText')
+        if (!columnExists) queryRunner.query(`ALTER TABLE \`chat_flow\` ADD COLUMN \`speechToText\` TEXT;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`chat_flow\` DROP COLUMN \`speechToText\`;`)
+    }
+}

packages/server/src/database/migrations/mysql/index.ts

@@ -10,7 +10,9 @@ import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEnt
 import { AddUsedToolsToChatMessage1699481607341 } from './1699481607341-AddUsedToolsToChatMessage'
 import { AddCategoryToChatFlow1699900910291 } from './1699900910291-AddCategoryToChatFlow'
 import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-AddFileAnnotationsToChatMessage'
+import { AddFileUploadsToChatMessage1701788586491 } from './1701788586491-AddFileUploadsToChatMessage'
 import { AddVariableEntity1699325775451 } from './1702200925471-AddVariableEntity'
+import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText'
 
 export const mysqlMigrations = [
     Init1693840429259,
@@ -25,5 +27,7 @@ export const mysqlMigrations = [
     AddUsedToolsToChatMessage1699481607341,
     AddCategoryToChatFlow1699900910291,
     AddFileAnnotationsToChatMessage1700271021237,
-    AddVariableEntity1699325775451
+    AddFileUploadsToChatMessage1701788586491,
+    AddVariableEntity1699325775451,
+    AddSpeechToText1706364937060
 ]

packages/server/src/database/migrations/postgres/1701788586491-AddFileUploadsToChatMessage.ts

@@ -0,0 +1,11 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddFileUploadsToChatMessage1701788586491 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "chat_message" ADD COLUMN IF NOT EXISTS "fileUploads" TEXT;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "chat_message" DROP COLUMN "fileUploads";`)
+    }
+}

packages/server/src/database/migrations/postgres/1706364937060-AddSpeechToText.ts

@@ -0,0 +1,11 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddSpeechToText1706364937060 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "chat_flow" ADD COLUMN IF NOT EXISTS "speechToText" TEXT;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "chat_flow" DROP COLUMN "speechToText";`)
+    }
+}

packages/server/src/database/migrations/postgres/index.ts

@@ -10,7 +10,9 @@ import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEnt
 import { AddUsedToolsToChatMessage1699481607341 } from './1699481607341-AddUsedToolsToChatMessage'
 import { AddCategoryToChatFlow1699900910291 } from './1699900910291-AddCategoryToChatFlow'
 import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-AddFileAnnotationsToChatMessage'
+import { AddFileUploadsToChatMessage1701788586491 } from './1701788586491-AddFileUploadsToChatMessage'
 import { AddVariableEntity1699325775451 } from './1702200925471-AddVariableEntity'
+import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText'
 
 export const postgresMigrations = [
     Init1693891895163,
@@ -25,5 +27,7 @@ export const postgresMigrations = [
     AddUsedToolsToChatMessage1699481607341,
     AddCategoryToChatFlow1699900910291,
     AddFileAnnotationsToChatMessage1700271021237,
-    AddVariableEntity1699325775451
+    AddFileUploadsToChatMessage1701788586491,
+    AddVariableEntity1699325775451,
+    AddSpeechToText1706364937060
 ]

packages/server/src/database/migrations/sqlite/1701788586491-AddFileUploadsToChatMessage.ts

@@ -0,0 +1,20 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddFileUploadsToChatMessage1701788586491 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE "temp_chat_message" ("id" varchar PRIMARY KEY NOT NULL, "role" varchar NOT NULL, "chatflowid" varchar NOT NULL, "content" text NOT NULL, "sourceDocuments" text, "usedTools" text, "fileAnnotations" text, "fileUploads" text, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "chatType" VARCHAR NOT NULL DEFAULT 'INTERNAL', "chatId" VARCHAR NOT NULL, "memoryType" VARCHAR, "sessionId" VARCHAR);`
+        )
+        await queryRunner.query(
+            `INSERT INTO "temp_chat_message" ("id", "role", "chatflowid", "content", "sourceDocuments", "fileAnnotations", "usedTools", "createdDate", "chatType", "chatId", "memoryType", "sessionId") SELECT "id", "role", "chatflowid", "content", "sourceDocuments", "usedTools", "fileAnnotations", "createdDate", "chatType", "chatId", "memoryType", "sessionId" FROM "chat_message";`
+        )
+        await queryRunner.query(`DROP TABLE "chat_message";`)
+        await queryRunner.query(`ALTER TABLE "temp_chat_message" RENAME TO "chat_message";`)
+        await queryRunner.query(`CREATE INDEX "IDX_e574527322272fd838f4f0f3d3" ON "chat_message" ("chatflowid") ;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE IF EXISTS "temp_chat_message";`)
+        await queryRunner.query(`ALTER TABLE "chat_message" DROP COLUMN "fileUploads";`)
+    }
+}

packages/server/src/database/migrations/sqlite/1706364937060-AddSpeechToText.ts

@@ -0,0 +1,11 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddSpeechToText1706364937060 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "chat_flow" ADD COLUMN "speechToText" TEXT;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "chat_flow" DROP COLUMN "speechToText";`)
+    }
+}

packages/server/src/database/migrations/sqlite/index.ts

@@ -10,7 +10,9 @@ import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEnt
 import { AddUsedToolsToChatMessage1699481607341 } from './1699481607341-AddUsedToolsToChatMessage'
 import { AddCategoryToChatFlow1699900910291 } from './1699900910291-AddCategoryToChatFlow'
 import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-AddFileAnnotationsToChatMessage'
+import { AddFileUploadsToChatMessage1701788586491 } from './1701788586491-AddFileUploadsToChatMessage'
 import { AddVariableEntity1699325775451 } from './1702200925471-AddVariableEntity'
+import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText'
 
 export const sqliteMigrations = [
     Init1693835579790,
@@ -25,5 +27,7 @@ export const sqliteMigrations = [
     AddUsedToolsToChatMessage1699481607341,
     AddCategoryToChatFlow1699900910291,
     AddFileAnnotationsToChatMessage1700271021237,
-    AddVariableEntity1699325775451
+    AddFileUploadsToChatMessage1701788586491,
+    AddVariableEntity1699325775451,
+    AddSpeechToText1706364937060
 ]

packages/server/src/index.ts

@@ -5,6 +5,7 @@ import cors from 'cors'
 import http from 'http'
 import * as fs from 'fs'
 import basicAuth from 'express-basic-auth'
+import contentDisposition from 'content-disposition'
 import { Server } from 'socket.io'
 import logger from './utils/logger'
 import { expressRequestLogger } from './utils/logger'
@@ -21,7 +22,8 @@ import {
     chatType,
     IChatMessage,
     IDepthQueue,
-    INodeDirectedGraph
+    INodeDirectedGraph,
+    IUploadFileSizeAndTypes
 } from './Interface'
 import {
     getNodeModulesPackagePath,
@@ -46,6 +48,7 @@ import {
     getAllConnectedNodes,
     clearSessionMemory,
     findMemoryNode,
+    deleteFolderRecursive,
     getTelemetryFlowObj,
     getAppVersion
 } from './utils'
@@ -59,7 +62,18 @@ import { Tool } from './database/entities/Tool'
 import { Assistant } from './database/entities/Assistant'
 import { ChatflowPool } from './ChatflowPool'
 import { CachePool } from './CachePool'
-import { ICommonObject, IMessage, INodeOptionsValue, handleEscapeCharacters, webCrawl, xmlScrape } from 'flowise-components'
+import {
+    ICommonObject,
+    IMessage,
+    INodeOptionsValue,
+    INodeParams,
+    handleEscapeCharacters,
+    convertSpeechToText,
+    xmlScrape,
+    webCrawl,
+    getStoragePath,
+    IFileUpload
+} from 'flowise-components'
 import { createRateLimiter, getRateLimiter, initializeRateLimiter } from './utils/rateLimit'
 import { addAPIKey, compareKeys, deleteAPIKey, getApiKey, getAPIKeys, updateAPIKey } from './utils/apiKey'
 import { sanitizeMiddleware, getCorsOptions, getAllowedIframeOrigins } from './utils/XSS'
@@ -85,7 +99,7 @@ export class App {
         // Initialize database
         this.AppDataSource.initialize()
             .then(async () => {
-                logger.info('📦 [server]: Data Source has been initialized!')
+                logger.info('📦 [server]: Data Source is being initialized!')
 
                 // Run Migrations Scripts
                 await this.AppDataSource.runMigrations({ transaction: 'each' })
@@ -112,6 +126,7 @@ export class App {
 
                 // Initialize telemetry
                 this.telemetry = new Telemetry()
+                logger.info('📦 [server]: Data Source has been initialized!')
             })
             .catch((err) => {
                 logger.error('❌ [server]: Error during Data Source initialization:', err)
@@ -167,7 +182,9 @@ export class App {
                 '/api/v1/node-icon/',
                 '/api/v1/components-credentials-icon/',
                 '/api/v1/chatflows-streaming',
+                '/api/v1/chatflows-uploads',
                 '/api/v1/openai-assistants-file',
+                '/api/v1/get-upload-file',
                 '/api/v1/ip'
             ]
             this.app.use((req, res, next) => {
@@ -307,7 +324,10 @@ export class App {
         // execute custom function node
         this.app.post('/api/v1/node-custom-function', async (req: Request, res: Response) => {
             const body = req.body
-            const nodeData = { inputs: body }
+            const functionInputVariables = Object.fromEntries(
+                [...(body?.javascriptFunction ?? '').matchAll(/\$([a-zA-Z0-9_]+)/g)].map((g) => [g[1], undefined])
+            )
+            const nodeData = { inputs: { functionInputVariables, ...body } }
             if (Object.prototype.hasOwnProperty.call(this.nodesPool.componentNodes, 'customFunction')) {
                 try {
                     const nodeInstanceFilePath = this.nodesPool.componentNodes['customFunction'].filePath as string
@@ -388,10 +408,13 @@ export class App {
                 id: req.params.id
             })
             if (!chatflow) return res.status(404).send(`Chatflow ${req.params.id} not found`)
-            if (chatflow.chatbotConfig) {
+            const uploadsConfig = await this.getUploadsConfig(req.params.id)
+            // even if chatbotConfig is not set but uploads are enabled
+            // send uploadsConfig to the chatbot
+            if (chatflow.chatbotConfig || uploadsConfig) {
                 try {
-                    const parsedConfig = JSON.parse(chatflow.chatbotConfig)
-                    return res.json(parsedConfig)
+                    const parsedConfig = chatflow.chatbotConfig ? JSON.parse(chatflow.chatbotConfig) : {}
+                    return res.json({ ...parsedConfig, uploads: uploadsConfig })
                 } catch (e) {
                     return res.status(500).send(`Error parsing Chatbot Config for Chatflow ${req.params.id}`)
                 }
@@ -451,6 +474,15 @@ export class App {
         // Delete chatflow via id
         this.app.delete('/api/v1/chatflows/:id', async (req: Request, res: Response) => {
             const results = await this.AppDataSource.getRepository(ChatFlow).delete({ id: req.params.id })
+
+            try {
+                // Delete all  uploads corresponding to this chatflow
+                const directory = path.join(getStoragePath(), req.params.id)
+                deleteFolderRecursive(directory)
+            } catch (e) {
+                logger.error(`[server]: Error deleting file storage for chatflow ${req.params.id}: ${e}`)
+            }
+
             return res.json(results)
         })
 
@@ -501,6 +533,16 @@ export class App {
             return res.json(obj)
         })
 
+        // Check if chatflow valid for uploads
+        this.app.get('/api/v1/chatflows-uploads/:id', async (req: Request, res: Response) => {
+            try {
+                const uploadsConfig = await this.getUploadsConfig(req.params.id)
+                return res.json(uploadsConfig)
+            } catch (e) {
+                return res.status(500).send(e)
+            }
+        })
+
         // ----------------------------------------
         // ChatMessage
         // ----------------------------------------
@@ -598,6 +640,16 @@ export class App {
             if (sessionId) deleteOptions.sessionId = sessionId
             if (chatType) deleteOptions.chatType = chatType
 
+            // Delete all uploads corresponding to this chatflow/chatId
+            if (chatId) {
+                try {
+                    const directory = path.join(getStoragePath(), chatflowid, chatId)
+                    deleteFolderRecursive(directory)
+                } catch (e) {
+                    logger.error(`[server]: Error deleting file storage for chatflow ${chatflowid}, chatId ${chatId}: ${e}`)
+                }
+            }
+
             const results = await this.AppDataSource.getRepository(ChatMessage).delete(deleteOptions)
             return res.json(results)
         })
@@ -1080,6 +1132,11 @@ export class App {
             }
         })
 
+        function streamFileToUser(res: Response, filePath: string) {
+            const fileStream = fs.createReadStream(filePath)
+            fileStream.pipe(res)
+        }
+
         // Download file from assistant
         this.app.post('/api/v1/openai-assistants-file', async (req: Request, res: Response) => {
             const filePath = path.join(getUserHome(), '.flowise', 'openai-assistant', req.body.fileName)
@@ -1089,9 +1146,48 @@ export class App {
             if (filePath.includes('..')) return res.status(500).send(`Invalid file path`)
             //only return from the .flowise openai-assistant folder
             if (!(filePath.includes('.flowise') && filePath.includes('openai-assistant'))) return res.status(500).send(`Invalid file path`)
-            res.setHeader('Content-Disposition', 'attachment; filename=' + path.basename(filePath))
-            const fileStream = fs.createReadStream(filePath)
-            fileStream.pipe(res)
+
+            if (fs.existsSync(filePath)) {
+                res.setHeader('Content-Disposition', contentDisposition(path.basename(filePath)))
+                streamFileToUser(res, filePath)
+            } else {
+                return res.status(404).send(`File ${req.body.fileName} not found`)
+            }
+        })
+
+        this.app.get('/api/v1/get-upload-path', async (req: Request, res: Response) => {
+            return res.json({
+                storagePath: getStoragePath()
+            })
+        })
+
+        // stream uploaded image
+        this.app.get('/api/v1/get-upload-file', async (req: Request, res: Response) => {
+            try {
+                if (!req.query.chatflowId || !req.query.chatId || !req.query.fileName) {
+                    return res.status(500).send(`Invalid file path`)
+                }
+                const chatflowId = req.query.chatflowId as string
+                const chatId = req.query.chatId as string
+                const fileName = req.query.fileName as string
+
+                const filePath = path.join(getStoragePath(), chatflowId, chatId, fileName)
+                //raise error if file path is not absolute
+                if (!path.isAbsolute(filePath)) return res.status(500).send(`Invalid file path`)
+                //raise error if file path contains '..'
+                if (filePath.includes('..')) return res.status(500).send(`Invalid file path`)
+                //only return from the storage folder
+                if (!filePath.startsWith(getStoragePath())) return res.status(500).send(`Invalid file path`)
+
+                if (fs.existsSync(filePath)) {
+                    res.setHeader('Content-Disposition', contentDisposition(path.basename(filePath)))
+                    streamFileToUser(res, filePath)
+                } else {
+                    return res.status(404).send(`File ${fileName} not found`)
+                }
+            } catch (error) {
+                return res.status(500).send(`Invalid file path`)
+            }
         })
 
         // ----------------------------------------
@@ -1149,18 +1245,22 @@ export class App {
         // ----------------------------------------
 
         this.app.get('/api/v1/fetch-links', async (req: Request, res: Response) => {
-            const url = decodeURIComponent(req.query.url as string)
-            const relativeLinksMethod = req.query.relativeLinksMethod as string
-            if (!relativeLinksMethod) {
-                return res.status(500).send('Please choose a Relative Links Method in Additional Parameters.')
+            try {
+                const url = decodeURIComponent(req.query.url as string)
+                const relativeLinksMethod = req.query.relativeLinksMethod as string
+                if (!relativeLinksMethod) {
+                    return res.status(500).send('Please choose a Relative Links Method in Additional Parameters.')
+                }
+
+                const limit = parseInt(req.query.limit as string)
+                if (process.env.DEBUG === 'true') console.info(`Start ${relativeLinksMethod}`)
+                const links: string[] = relativeLinksMethod === 'webCrawl' ? await webCrawl(url, limit) : await xmlScrape(url, limit)
+                if (process.env.DEBUG === 'true') console.info(`Finish ${relativeLinksMethod}`)
+
+                res.json({ status: 'OK', links })
+            } catch (e: any) {
+                return res.status(500).send('Could not fetch links from the URL.')
             }
-
-            const limit = parseInt(req.query.limit as string)
-            if (process.env.DEBUG === 'true') console.info(`Start ${relativeLinksMethod}`)
-            const links: string[] = relativeLinksMethod === 'webCrawl' ? await webCrawl(url, limit) : await xmlScrape(url, limit)
-            if (process.env.DEBUG === 'true') console.info(`Finish ${relativeLinksMethod}`)
-
-            res.json({ status: 'OK', links })
         })
 
         // ----------------------------------------
@@ -1444,6 +1544,74 @@ export class App {
         return false
     }
 
+    /**
+     * Method that checks if uploads are enabled in the chatflow
+     * @param {string} chatflowid
+     */
+    async getUploadsConfig(chatflowid: string): Promise<any> {
+        const chatflow = await this.AppDataSource.getRepository(ChatFlow).findOneBy({
+            id: chatflowid
+        })
+        if (!chatflow) return `Chatflow ${chatflowid} not found`
+
+        const uploadAllowedNodes = ['llmChain', 'conversationChain', 'mrklAgentChat', 'conversationalAgent']
+        const uploadProcessingNodes = ['chatOpenAI']
+
+        const flowObj = JSON.parse(chatflow.flowData)
+        const imgUploadSizeAndTypes: IUploadFileSizeAndTypes[] = []
+
+        let isSpeechToTextEnabled = false
+        if (chatflow.speechToText) {
+            const speechToTextProviders = JSON.parse(chatflow.speechToText)
+            for (const provider in speechToTextProviders) {
+                if (provider !== 'none') {
+                    const providerObj = speechToTextProviders[provider]
+                    if (providerObj.status) {
+                        isSpeechToTextEnabled = true
+                        break
+                    }
+                }
+            }
+        }
+
+        let isImageUploadAllowed = false
+        const nodes: IReactFlowNode[] = flowObj.nodes
+
+        /*
+         * Condition for isImageUploadAllowed
+         * 1.) one of the uploadAllowedNodes exists
+         * 2.) one of the uploadProcessingNodes exists + allowImageUploads is ON
+         */
+        if (!nodes.some((node) => uploadAllowedNodes.includes(node.data.name))) {
+            return {
+                isSpeechToTextEnabled,
+                isImageUploadAllowed: false,
+                imgUploadSizeAndTypes
+            }
+        }
+
+        nodes.forEach((node: IReactFlowNode) => {
+            if (uploadProcessingNodes.indexOf(node.data.name) > -1) {
+                // TODO: for now the maxUploadSize is hardcoded to 5MB, we need to add it to the node properties
+                node.data.inputParams.map((param: INodeParams) => {
+                    if (param.name === 'allowImageUploads' && node.data.inputs?.['allowImageUploads']) {
+                        imgUploadSizeAndTypes.push({
+                            fileTypes: 'image/gif;image/jpeg;image/png;image/webp;'.split(';'),
+                            maxUploadSize: 5
+                        })
+                        isImageUploadAllowed = true
+                    }
+                })
+            }
+        })
+
+        return {
+            isSpeechToTextEnabled,
+            isImageUploadAllowed,
+            imgUploadSizeAndTypes
+        }
+    }
+
     /**
      * Method that get chat messages.
      * @param {string} chatflowid
@@ -1657,6 +1825,57 @@ export class App {
                 if (!isKeyValidated) return res.status(401).send('Unauthorized')
             }
 
+            let fileUploads: IFileUpload[] = []
+            if (incomingInput.uploads) {
+                fileUploads = incomingInput.uploads
+                for (let i = 0; i < fileUploads.length; i += 1) {
+                    const upload = fileUploads[i]
+                    if ((upload.type === 'file' || upload.type === 'audio') && upload.data) {
+                        const filename = upload.name
+                        const dir = path.join(getStoragePath(), chatflowid, chatId)
+                        if (!fs.existsSync(dir)) {
+                            fs.mkdirSync(dir, { recursive: true })
+                        }
+                        const filePath = path.join(dir, filename)
+                        const splitDataURI = upload.data.split(',')
+                        const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
+                        fs.writeFileSync(filePath, bf)
+
+                        // Omit upload.data since we don't store the content in database
+                        upload.type = 'stored-file'
+                        fileUploads[i] = omit(upload, ['data'])
+                    }
+
+                    // Run Speech to Text conversion
+                    if (upload.mime === 'audio/webm') {
+                        let speechToTextConfig: ICommonObject = {}
+                        if (chatflow.speechToText) {
+                            const speechToTextProviders = JSON.parse(chatflow.speechToText)
+                            for (const provider in speechToTextProviders) {
+                                const providerObj = speechToTextProviders[provider]
+                                if (providerObj.status) {
+                                    speechToTextConfig = providerObj
+                                    speechToTextConfig['name'] = provider
+                                    break
+                                }
+                            }
+                        }
+                        if (speechToTextConfig) {
+                            const options: ICommonObject = {
+                                chatId,
+                                chatflowid,
+                                appDataSource: this.AppDataSource,
+                                databaseEntities: databaseEntities
+                            }
+                            const speechToTextResult = await convertSpeechToText(upload, speechToTextConfig, options)
+                            if (speechToTextResult) {
+                                incomingInput.question = speechToTextResult
+                            }
+                        }
+                    }
+                }
+            }
+
             let isStreamValid = false
 
             const files = (req.files as any[]) || []
@@ -1822,7 +2041,10 @@ export class App {
                     chatflowid,
                     this.AppDataSource,
                     incomingInput?.overrideConfig,
-                    this.cachePool
+                    this.cachePool,
+                    false,
+                    undefined,
+                    incomingInput.uploads
                 )
 
                 const nodeToExecute =
@@ -1861,6 +2083,7 @@ export class App {
                       appDataSource: this.AppDataSource,
                       databaseEntities,
                       analytic: chatflow.analytic,
+                      uploads: incomingInput.uploads,
                       socketIO,
                       socketIOClientId: incomingInput.socketIOClientId
                   })
@@ -1871,7 +2094,8 @@ export class App {
                       logger,
                       appDataSource: this.AppDataSource,
                       databaseEntities,
-                      analytic: chatflow.analytic
+                      analytic: chatflow.analytic,
+                      uploads: incomingInput.uploads
                   })
 
             result = typeof result === 'string' ? { text: result } : result
@@ -1889,7 +2113,8 @@ export class App {
                 chatId,
                 memoryType,
                 sessionId,
-                createdDate: userMessageDateTime
+                createdDate: userMessageDateTime,
+                fileUploads: incomingInput.uploads ? JSON.stringify(fileUploads) : undefined
             }
             await this.addChatMessage(userMessage)
 
@@ -1911,7 +2136,6 @@ export class App {
             if (result?.usedTools) apiMessage.usedTools = JSON.stringify(result.usedTools)
             if (result?.fileAnnotations) apiMessage.fileAnnotations = JSON.stringify(result.fileAnnotations)
             const chatMessage = await this.addChatMessage(apiMessage)
-            result.chatMessageId = chatMessage.id
 
             logger.debug(`[server]: Finished running ${nodeToExecuteData.label} (${nodeToExecuteData.id})`)
             await this.telemetry.sendTelemetry('prediction_sent', {
@@ -1923,7 +2147,11 @@ export class App {
             })
 
             // Prepare response
+            // return the question in the response
+            // this is used when input text is empty but question is in audio format
+            result.question = incomingInput.question
             result.chatId = chatId
+            result.chatMessageId = chatMessage.id
             if (sessionId) result.sessionId = sessionId
             if (memoryType) result.memoryType = memoryType
 

packages/server/src/utils/index.ts

@@ -27,7 +27,8 @@ import {
     ICommonObject,
     IDatabaseEntity,
     IMessage,
-    FlowiseMemory
+    FlowiseMemory,
+    IFileUpload
 } from 'flowise-components'
 import { randomBytes } from 'crypto'
 import { AES, enc } from 'crypto-js'
@@ -279,7 +280,8 @@ export const buildFlow = async (
     overrideConfig?: ICommonObject,
     cachePool?: CachePool,
     isUpsert?: boolean,
-    stopNodeId?: string
+    stopNodeId?: string,
+    uploads?: IFileUpload[]
 ) => {
     const flowNodes = cloneDeep(reactFlowNodes)
 
@@ -325,7 +327,8 @@ export const buildFlow = async (
                     appDataSource,
                     databaseEntities,
                     cachePool,
-                    dynamicVariables
+                    dynamicVariables,
+                    uploads
                 })
                 logger.debug(`[server]: Finished upserting ${reactFlowNode.data.label} (${reactFlowNode.data.id})`)
                 break
@@ -340,7 +343,8 @@ export const buildFlow = async (
                     appDataSource,
                     databaseEntities,
                     cachePool,
-                    dynamicVariables
+                    dynamicVariables,
+                    uploads
                 })
 
                 // Save dynamic variables
@@ -596,7 +600,6 @@ export const resolveVariables = (
     }
 
     const paramsObj = flowNodeData[types] ?? {}
-
     getParamValues(paramsObj)
 
     return flowNodeData
@@ -629,7 +632,33 @@ export const replaceInputsWithConfig = (flowNodeData: INodeData, overrideConfig:
                 }
             }
 
-            let paramValue = overrideConfig[config] ?? inputsObj[config]
+            let paramValue = inputsObj[config]
+            const overrideConfigValue = overrideConfig[config]
+            if (overrideConfigValue) {
+                if (typeof overrideConfigValue === 'object') {
+                    switch (typeof paramValue) {
+                        case 'string':
+                            if (paramValue.startsWith('{') && paramValue.endsWith('}')) {
+                                try {
+                                    paramValue = Object.assign({}, JSON.parse(paramValue), overrideConfigValue)
+                                    break
+                                } catch (e) {
+                                    // ignore
+                                }
+                            }
+                            paramValue = overrideConfigValue
+                            break
+                        case 'object':
+                            paramValue = Object.assign({}, paramValue, overrideConfigValue)
+                            break
+                        default:
+                            paramValue = overrideConfigValue
+                            break
+                    }
+                } else {
+                    paramValue = overrideConfigValue
+                }
+            }
             // Check if boolean
             if (paramValue === 'true') paramValue = true
             else if (paramValue === 'false') paramValue = false
@@ -1102,6 +1131,34 @@ export const getAllValuesFromJson = (obj: any): any[] => {
     return values
 }
 
+/**
+ * Delete file & folder recursively
+ * @param {string} directory
+ */
+export const deleteFolderRecursive = (directory: string) => {
+    if (fs.existsSync(directory)) {
+        fs.readdir(directory, (error, files) => {
+            if (error) throw new Error('Could not read directory')
+
+            files.forEach((file) => {
+                const file_path = path.join(directory, file)
+
+                fs.stat(file_path, (error, stat) => {
+                    if (error) throw new Error('File do not exist')
+
+                    if (!stat.isDirectory()) {
+                        fs.unlink(file_path, (error) => {
+                            if (error) throw new Error('Could not delete file')
+                        })
+                    } else {
+                        deleteFolderRecursive(file_path)
+                    }
+                })
+            })
+        })
+    }
+}
+
 /**
  * Get only essential flow data items for telemetry
  * @param {IReactFlowNode[]} nodes

packages/ui/package.json

@@ -1,6 +1,6 @@
 {
     "name": "flowise-ui",
-    "version": "1.5.0",
+    "version": "1.5.1",
     "license": "SEE LICENSE IN LICENSE.md",
     "homepage": "https://flowiseai.com",
     "author": {

packages/ui/src/api/chatflows.js

@@ -14,6 +14,8 @@ const deleteChatflow = (id) => client.delete(`/chatflows/${id}`)
 
 const getIsChatflowStreaming = (id) => client.get(`/chatflows-streaming/${id}`)
 
+const getAllowChatflowUploads = (id) => client.get(`/chatflows-uploads/${id}`)
+
 export default {
     getAllChatflows,
     getSpecificChatflow,
@@ -21,5 +23,6 @@ export default {
     createNewChatflow,
     updateChatflow,
     deleteChatflow,
-    getIsChatflowStreaming
+    getIsChatflowStreaming,
+    getAllowChatflowUploads
 }

packages/ui/src/api/chatmessage.js

@@ -4,10 +4,12 @@ const getInternalChatmessageFromChatflow = (id) => client.get(`/internal-chatmes
 const getAllChatmessageFromChatflow = (id, params = {}) => client.get(`/chatmessage/${id}`, { params: { order: 'DESC', ...params } })
 const getChatmessageFromPK = (id, params = {}) => client.get(`/chatmessage/${id}`, { params: { order: 'ASC', ...params } })
 const deleteChatmessage = (id, params = {}) => client.delete(`/chatmessage/${id}`, { params: { ...params } })
+const getStoragePath = () => client.get(`/get-upload-path`)
 
 export default {
     getInternalChatmessageFromChatflow,
     getAllChatmessageFromChatflow,
     getChatmessageFromPK,
-    deleteChatmessage
+    deleteChatmessage,
+    getStoragePath
 }

packages/ui/src/assets/images/openai.svg

@@ -0,0 +1,8 @@
+<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M14.9814 25.2665C15.353 26.2672 16.0645 27.1054 16.9914 27.6347C17.9183 28.164 19.0018 28.3507 20.0524 28.1622C21.103 27.9737 22.054 27.422 22.7391 26.6034C23.4242 25.7849 23.7998 24.7517 23.8004 23.6842V17.5533C23.8004 17.1909 23.6043 16.8569 23.2879 16.6802L15.9995 12.6108" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M7.17701 19.5848C6.49568 20.4069 6.12505 21.4424 6.12993 22.5101C6.13481 23.5779 6.51489 24.6099 7.2037 25.4258C7.89252 26.2416 8.84622 26.7893 9.89802 26.9732C10.9498 27.157 12.0328 26.9653 12.9575 26.4314L18.1044 23.4263C18.4114 23.247 18.6002 22.9182 18.6002 22.5627V14.106" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M8.19877 9.98459C6.39026 9.67775 4.57524 10.4982 3.60403 12.1806C3.00524 13.2178 2.84295 14.4504 3.15284 15.6073C3.46273 16.7642 4.21943 17.7507 5.25652 18.3498L10.3049 21.3269C10.6109 21.5074 10.9898 21.5119 11.3001 21.3388L18.6 17.2655" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M17.0172 6.06585C16.6456 5.06522 15.9342 4.227 15.0072 3.6977C14.0803 3.1684 12.9969 2.98168 11.9462 3.17018C10.8956 3.35869 9.94464 3.91042 9.25954 4.72895C8.57444 5.54747 8.19879 6.58074 8.19824 7.64814V13.6575C8.19824 14.0154 8.38951 14.346 8.69977 14.5244L15.9992 18.7215" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M24.8216 11.7476C25.5029 10.9255 25.8735 9.89004 25.8687 8.8223C25.8638 7.75457 25.4837 6.72253 24.7949 5.90667C24.1061 5.09082 23.1524 4.54308 22.1006 4.35924C21.0488 4.17541 19.9658 4.36718 19.0411 4.90101L13.8942 7.90613C13.5872 8.08539 13.3984 8.41418 13.3984 8.76971V17.2265" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M23.7997 21.2595C25.6082 21.5663 27.4232 20.7459 28.3944 19.0635C28.9932 18.0263 29.1555 16.7937 28.8456 15.6368C28.5357 14.4799 27.779 13.4934 26.7419 12.8943L21.6409 9.91752C21.3316 9.73703 20.9494 9.7357 20.6388 9.91405L17.2696 11.849L13.3984 14.0723" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>

packages/ui/src/menu-items/settings.js

@@ -7,11 +7,22 @@ import {
     IconSearch,
     IconMessage,
     IconPictureInPictureOff,
-    IconLink
+    IconLink,
+    IconMicrophone
 } from '@tabler/icons'
 
 // constant
-const icons = { IconTrash, IconFileUpload, IconFileExport, IconCopy, IconSearch, IconMessage, IconPictureInPictureOff, IconLink }
+const icons = {
+    IconTrash,
+    IconFileUpload,
+    IconFileExport,
+    IconCopy,
+    IconSearch,
+    IconMessage,
+    IconPictureInPictureOff,
+    IconLink,
+    IconMicrophone
+}
 
 // ==============================|| SETTINGS MENU ITEMS ||============================== //
 
@@ -41,6 +52,13 @@ const settings = {
             url: '',
             icon: icons.IconLink
         },
+        {
+            id: 'enableSpeechToText',
+            title: 'Speech to Text',
+            type: 'item',
+            url: '',
+            icon: icons.IconMicrophone
+        },
         {
             id: 'duplicateChatflow',
             title: 'Duplicate Chatflow',

packages/ui/src/ui-component/button/ImageButton.js

@@ -0,0 +1,57 @@
+import { styled } from '@mui/material/styles'
+import ButtonBase from '@mui/material/ButtonBase'
+
+export const ImageButton = styled(ButtonBase)(({ theme }) => ({
+    position: 'relative',
+    height: 200,
+    borderRadius: '10px',
+    [theme.breakpoints.down('sm')]: {
+        width: '100% !important', // Overrides inline-style
+        height: 100
+    },
+    '&:hover, &.Mui-focusVisible': {
+        zIndex: 1,
+        '& .MuiImageBackdrop-root': {
+            opacity: 0.4
+        },
+        '& .MuiImageMarked-root': {
+            opacity: 1
+        },
+        '& .MuiTypography-root': {
+            border: '4px solid currentColor'
+        }
+    }
+}))
+
+export const ImageSrc = styled('span')({
+    position: 'absolute',
+    borderRadius: '10px',
+    left: 0,
+    right: 0,
+    top: 0,
+    bottom: 0,
+    backgroundSize: 'cover',
+    backgroundPosition: 'center 40%'
+})
+
+export const ImageBackdrop = styled('span')(({ theme }) => ({
+    position: 'absolute',
+    borderRadius: '10px',
+    left: 0,
+    right: 0,
+    top: 0,
+    bottom: 0,
+    backgroundColor: theme.palette.common.black,
+    opacity: 0.1,
+    transition: theme.transitions.create('opacity')
+}))
+
+export const ImageMarked = styled('span')(() => ({
+    height: 25,
+    width: 25,
+    backgroundColor: 'transparent',
+    position: 'absolute',
+    top: 'auto',
+    left: 'auto',
+    opacity: 0
+}))

packages/ui/src/ui-component/cards/StarterPromptsCard.css

@@ -1,7 +1,4 @@
 .button-container {
-    position: absolute;
-    bottom: 0;
-    z-index: 1000;
     display: flex;
     overflow-x: auto;
     -webkit-overflow-scrolling: touch; /* For momentum scroll on mobile devices */
@@ -10,5 +7,4 @@
 
 .button {
     flex: 0 0 auto; /* Don't grow, don't shrink, base width on content */
-    margin: 5px; /* Adjust as needed for spacing between buttons */
 }

packages/ui/src/ui-component/cards/StarterPromptsCard.js

@@ -3,9 +3,12 @@ import PropTypes from 'prop-types'
 import { Chip } from '@mui/material'
 import './StarterPromptsCard.css'
 
-const StarterPromptsCard = ({ isGrid, starterPrompts, onPromptClick }) => {
+const StarterPromptsCard = ({ isGrid, starterPrompts, sx, onPromptClick }) => {
     return (
-        <Box className={'button-container'} sx={{ maxWidth: isGrid ? 'inherit' : '400px', m: 1 }}>
+        <Box
+            className={'button-container'}
+            sx={{ width: '100%', maxWidth: isGrid ? 'inherit' : '400px', p: 1.5, display: 'flex', gap: 1, ...sx }}
+        >
             {starterPrompts.map((sp, index) => (
                 <Chip label={sp.prompt} className={'button'} key={index} onClick={(e) => onPromptClick(sp.prompt, e)} />
             ))}
@@ -15,7 +18,8 @@ const StarterPromptsCard = ({ isGrid, starterPrompts, onPromptClick }) => {
 
 StarterPromptsCard.propTypes = {
     isGrid: PropTypes.bool,
-    starterPrompts: PropTypes.arrayOf(PropTypes.string),
+    starterPrompts: PropTypes.array,
+    sx: PropTypes.object,
     onPromptClick: PropTypes.func
 }
 

packages/ui/src/ui-component/dialog/ManageScrapedLinksDialog.js

@@ -16,7 +16,7 @@ import {
     Stack,
     Typography
 } from '@mui/material'
-import { IconTrash, IconX } from '@tabler/icons'
+import { IconEraser, IconTrash, IconX } from '@tabler/icons'
 import PerfectScrollbar from 'react-perfect-scrollbar'
 
 import { BackdropLoader } from 'ui-component/loading/BackdropLoader'
@@ -113,6 +113,10 @@ const ManageScrapedLinksDialog = ({ show, dialogProps, onCancel, onSave }) => {
         setSelectedLinks(links)
     }
 
+    const handleRemoveAllLinks = () => {
+        setSelectedLinks([])
+    }
+
     const handleSaveLinks = () => {
         onSave(url, selectedLinks)
     }
@@ -145,6 +149,7 @@ const ManageScrapedLinksDialog = ({ show, dialogProps, onCancel, onSave }) => {
                             />
                         </FormControl>
                         <Button
+                            disabled={!url}
                             sx={{ borderRadius: '12px', mt: 1, display: 'flex', flexShrink: 0 }}
                             size='small'
                             variant='contained'
@@ -154,7 +159,21 @@ const ManageScrapedLinksDialog = ({ show, dialogProps, onCancel, onSave }) => {
                         </Button>
                     </Stack>
                 </Box>
-                <Typography sx={{ mb: 2, fontWeight: 500 }}>Scraped Links</Typography>
+                <Box sx={{ display: 'flex', alignItems: 'center', justifyContent: 'space-between', mb: 1.5 }}>
+                    <Typography sx={{ fontWeight: 500 }}>Scraped Links</Typography>
+                    {selectedLinks.length > 0 ? (
+                        <StyledButton
+                            sx={{ height: 'max-content', width: 'max-content' }}
+                            variant='outlined'
+                            color='error'
+                            title='Clear All Links'
+                            onClick={handleRemoveAllLinks}
+                            startIcon={<IconEraser />}
+                        >
+                            Clear All
+                        </StyledButton>
+                    ) : null}
+                </Box>
                 <>
                     {loading && <BackdropLoader open={loading} />}
                     {selectedLinks.length > 0 ? (

packages/ui/src/ui-component/dialog/SpeechToTextDialog.js

@@ -0,0 +1,348 @@
+import { createPortal } from 'react-dom'
+import { useDispatch } from 'react-redux'
+import { useState, useEffect } from 'react'
+import PropTypes from 'prop-types'
+import { enqueueSnackbar as enqueueSnackbarAction, closeSnackbar as closeSnackbarAction, SET_CHATFLOW } from 'store/actions'
+
+// material-ui
+import {
+    Typography,
+    Box,
+    Button,
+    Dialog,
+    DialogContent,
+    DialogTitle,
+    DialogActions,
+    FormControl,
+    ListItem,
+    ListItemAvatar,
+    ListItemText,
+    MenuItem,
+    Select
+} from '@mui/material'
+import { IconX } from '@tabler/icons'
+
+// Project import
+import CredentialInputHandler from 'views/canvas/CredentialInputHandler'
+import { TooltipWithParser } from 'ui-component/tooltip/TooltipWithParser'
+import { SwitchInput } from 'ui-component/switch/Switch'
+import { Input } from 'ui-component/input/Input'
+import { StyledButton } from 'ui-component/button/StyledButton'
+import { Dropdown } from 'ui-component/dropdown/Dropdown'
+import openAISVG from 'assets/images/openai.svg'
+import assemblyAIPng from 'assets/images/assemblyai.png'
+
+// store
+import { HIDE_CANVAS_DIALOG, SHOW_CANVAS_DIALOG } from 'store/actions'
+import useNotifier from 'utils/useNotifier'
+
+// API
+import chatflowsApi from 'api/chatflows'
+
+const speechToTextProviders = {
+    openAIWhisper: {
+        label: 'OpenAI Whisper',
+        name: 'openAIWhisper',
+        icon: openAISVG,
+        url: 'https://platform.openai.com/docs/guides/speech-to-text',
+        inputs: [
+            {
+                label: 'Connect Credential',
+                name: 'credential',
+                type: 'credential',
+                credentialNames: ['openAIApi']
+            },
+            {
+                label: 'Language',
+                name: 'language',
+                type: 'string',
+                description:
+                    'The language of the input audio. Supplying the input language in ISO-639-1 format will improve accuracy and latency.',
+                placeholder: 'en',
+                optional: true
+            },
+            {
+                label: 'Prompt',
+                name: 'prompt',
+                type: 'string',
+                rows: 4,
+                description: `An optional text to guide the model's style or continue a previous audio segment. The prompt should match the audio language.`,
+                optional: true
+            },
+            {
+                label: 'Temperature',
+                name: 'temperature',
+                type: 'number',
+                step: 0.1,
+                description: `The sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.`,
+                optional: true
+            }
+        ]
+    },
+    assemblyAiTranscribe: {
+        label: 'Assembly AI',
+        name: 'assemblyAiTranscribe',
+        icon: assemblyAIPng,
+        url: 'https://www.assemblyai.com/',
+        inputs: [
+            {
+                label: 'Connect Credential',
+                name: 'credential',
+                type: 'credential',
+                credentialNames: ['assemblyAIApi']
+            }
+        ]
+    }
+}
+
+const SpeechToTextDialog = ({ show, dialogProps, onCancel }) => {
+    const portalElement = document.getElementById('portal')
+    const dispatch = useDispatch()
+
+    useNotifier()
+
+    const enqueueSnackbar = (...args) => dispatch(enqueueSnackbarAction(...args))
+    const closeSnackbar = (...args) => dispatch(closeSnackbarAction(...args))
+
+    const [speechToText, setSpeechToText] = useState({})
+    const [selectedProvider, setSelectedProvider] = useState('none')
+
+    const onSave = async () => {
+        const speechToText = setValue(true, selectedProvider, 'status')
+        try {
+            const saveResp = await chatflowsApi.updateChatflow(dialogProps.chatflow.id, {
+                speechToText: JSON.stringify(speechToText)
+            })
+            if (saveResp.data) {
+                enqueueSnackbar({
+                    message: 'Speech To Text Configuration Saved',
+                    options: {
+                        key: new Date().getTime() + Math.random(),
+                        variant: 'success',
+                        action: (key) => (
+                            <Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
+                                <IconX />
+                            </Button>
+                        )
+                    }
+                })
+                dispatch({ type: SET_CHATFLOW, chatflow: saveResp.data })
+            }
+            onCancel()
+        } catch (error) {
+            const errorData = error.response.data || `${error.response.status}: ${error.response.statusText}`
+            enqueueSnackbar({
+                message: `Failed to save Speech To Text Configuration: ${errorData}`,
+                options: {
+                    key: new Date().getTime() + Math.random(),
+                    variant: 'error',
+                    persist: true,
+                    action: (key) => (
+                        <Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
+                            <IconX />
+                        </Button>
+                    )
+                }
+            })
+        }
+    }
+
+    const setValue = (value, providerName, inputParamName) => {
+        let newVal = {}
+        if (!Object.prototype.hasOwnProperty.call(speechToText, providerName)) {
+            newVal = { ...speechToText, [providerName]: {} }
+        } else {
+            newVal = { ...speechToText }
+        }
+
+        newVal[providerName][inputParamName] = value
+        if (inputParamName === 'status' && value === true) {
+            // ensure that the others are turned off
+            Object.keys(speechToTextProviders).forEach((key) => {
+                const provider = speechToTextProviders[key]
+                if (provider.name !== providerName) {
+                    newVal[provider.name] = { ...speechToText[provider.name], status: false }
+                }
+            })
+        }
+        setSpeechToText(newVal)
+        return newVal
+    }
+
+    const handleProviderChange = (event) => {
+        setSelectedProvider(event.target.value)
+    }
+
+    useEffect(() => {
+        if (dialogProps.chatflow && dialogProps.chatflow.speechToText) {
+            try {
+                const speechToText = JSON.parse(dialogProps.chatflow.speechToText)
+                let selectedProvider = 'none'
+                Object.keys(speechToTextProviders).forEach((key) => {
+                    const providerConfig = speechToText[key]
+                    if (providerConfig && providerConfig.status) {
+                        selectedProvider = key
+                    }
+                })
+                setSelectedProvider(selectedProvider)
+                setSpeechToText(speechToText)
+            } catch (e) {
+                setSpeechToText({})
+                setSelectedProvider('none')
+                console.error(e)
+            }
+        }
+
+        return () => {
+            setSpeechToText({})
+            setSelectedProvider('none')
+        }
+    }, [dialogProps])
+
+    useEffect(() => {
+        if (show) dispatch({ type: SHOW_CANVAS_DIALOG })
+        else dispatch({ type: HIDE_CANVAS_DIALOG })
+        return () => dispatch({ type: HIDE_CANVAS_DIALOG })
+    }, [show, dispatch])
+
+    const component = (
+        <Dialog
+            onClose={onCancel}
+            open={show}
+            fullWidth
+            maxWidth='sm'
+            aria-labelledby='alert-dialog-title'
+            aria-describedby='alert-dialog-description'
+        >
+            <DialogTitle sx={{ fontSize: '1rem' }} id='alert-dialog-title'>
+                Speech To Text Configuration
+            </DialogTitle>
+            <DialogContent>
+                <Box fullWidth sx={{ my: 2, display: 'flex', flexDirection: 'column', gap: 1 }}>
+                    <Typography>Speech To Text Providers</Typography>
+                    <FormControl fullWidth>
+                        <Select value={selectedProvider} onChange={handleProviderChange}>
+                            <MenuItem value='none'>None</MenuItem>
+                            <MenuItem value='openAIWhisper'>OpenAI Whisper</MenuItem>
+                            <MenuItem value='assemblyAiTranscribe'>Assembly AI</MenuItem>
+                        </Select>
+                    </FormControl>
+                </Box>
+                {selectedProvider !== 'none' && (
+                    <>
+                        <ListItem style={{ padding: 0, margin: 0 }} alignItems='center'>
+                            <ListItemAvatar>
+                                <div
+                                    style={{
+                                        width: 50,
+                                        height: 50,
+                                        borderRadius: '50%',
+                                        backgroundColor: 'white'
+                                    }}
+                                >
+                                    <img
+                                        style={{
+                                            width: '100%',
+                                            height: '100%',
+                                            padding: 10,
+                                            objectFit: 'contain'
+                                        }}
+                                        alt='AI'
+                                        src={speechToTextProviders[selectedProvider].icon}
+                                    />
+                                </div>
+                            </ListItemAvatar>
+                            <ListItemText
+                                sx={{ ml: 1 }}
+                                primary={speechToTextProviders[selectedProvider].label}
+                                secondary={
+                                    <a target='_blank' rel='noreferrer' href={speechToTextProviders[selectedProvider].url}>
+                                        {speechToTextProviders[selectedProvider].url}
+                                    </a>
+                                }
+                            />
+                        </ListItem>
+                        {speechToTextProviders[selectedProvider].inputs.map((inputParam, index) => (
+                            <Box key={index} sx={{ p: 2 }}>
+                                <div style={{ display: 'flex', flexDirection: 'row' }}>
+                                    <Typography>
+                                        {inputParam.label}
+                                        {!inputParam.optional && <span style={{ color: 'red' }}>&nbsp;*</span>}
+                                        {inputParam.description && (
+                                            <TooltipWithParser style={{ marginLeft: 10 }} title={inputParam.description} />
+                                        )}
+                                    </Typography>
+                                </div>
+                                {inputParam.type === 'credential' && (
+                                    <CredentialInputHandler
+                                        key={speechToText[selectedProvider]?.credentialId}
+                                        data={
+                                            speechToText[selectedProvider]?.credentialId
+                                                ? { credential: speechToText[selectedProvider].credentialId }
+                                                : {}
+                                        }
+                                        inputParam={inputParam}
+                                        onSelect={(newValue) => setValue(newValue, selectedProvider, 'credentialId')}
+                                    />
+                                )}
+                                {inputParam.type === 'boolean' && (
+                                    <SwitchInput
+                                        onChange={(newValue) => setValue(newValue, selectedProvider, inputParam.name)}
+                                        value={
+                                            speechToText[selectedProvider]
+                                                ? speechToText[selectedProvider][inputParam.name]
+                                                : inputParam.default ?? false
+                                        }
+                                    />
+                                )}
+                                {(inputParam.type === 'string' || inputParam.type === 'password' || inputParam.type === 'number') && (
+                                    <Input
+                                        inputParam={inputParam}
+                                        onChange={(newValue) => setValue(newValue, selectedProvider, inputParam.name)}
+                                        value={
+                                            speechToText[selectedProvider]
+                                                ? speechToText[selectedProvider][inputParam.name]
+                                                : inputParam.default ?? ''
+                                        }
+                                    />
+                                )}
+
+                                {inputParam.type === 'options' && (
+                                    <Dropdown
+                                        name={inputParam.name}
+                                        options={inputParam.options}
+                                        onSelect={(newValue) => setValue(newValue, selectedProvider, inputParam.name)}
+                                        value={
+                                            speechToText[selectedProvider]
+                                                ? speechToText[selectedProvider][inputParam.name]
+                                                : inputParam.default ?? 'choose an option'
+                                        }
+                                    />
+                                )}
+                            </Box>
+                        ))}
+                    </>
+                )}
+            </DialogContent>
+            <DialogActions>
+                <StyledButton
+                    disabled={selectedProvider !== 'none' && !speechToText[selectedProvider]?.credentialId}
+                    variant='contained'
+                    onClick={onSave}
+                >
+                    Save
+                </StyledButton>
+            </DialogActions>
+        </Dialog>
+    )
+
+    return createPortal(component, portalElement)
+}
+
+SpeechToTextDialog.propTypes = {
+    show: PropTypes.bool,
+    dialogProps: PropTypes.object,
+    onCancel: PropTypes.func
+}
+
+export default SpeechToTextDialog

packages/ui/src/ui-component/dialog/ViewMessagesDialog.js

@@ -21,7 +21,9 @@ import {
     DialogTitle,
     ListItem,
     ListItemText,
-    Chip
+    Chip,
+    Card,
+    CardMedia
 } from '@mui/material'
 import { useTheme } from '@mui/material/styles'
 import DatePicker from 'react-datepicker'
@@ -47,7 +49,7 @@ import useApi from 'hooks/useApi'
 import useConfirm from 'hooks/useConfirm'
 
 // Utils
-import { isValidURL, removeDuplicateURL } from 'utils/genericHelper'
+import { getOS, isValidURL, removeDuplicateURL } from 'utils/genericHelper'
 import useNotifier from 'utils/useNotifier'
 import { baseURL } from 'store/constant'
 
@@ -69,6 +71,12 @@ DatePickerCustomInput.propTypes = {
     onClick: PropTypes.func
 }
 
+const messageImageStyle = {
+    width: '128px',
+    height: '128px',
+    objectFit: 'cover'
+}
+
 const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
     const portalElement = document.getElementById('portal')
     const dispatch = useDispatch()
@@ -92,6 +100,8 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
 
     const getChatmessageApi = useApi(chatmessageApi.getAllChatmessageFromChatflow)
     const getChatmessageFromPKApi = useApi(chatmessageApi.getChatmessageFromPK)
+    const getStoragePathFromServer = useApi(chatmessageApi.getStoragePath)
+    let storagePath = ''
 
     const onStartDateSelected = (date) => {
         setStartDate(date)
@@ -120,16 +130,35 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
         })
     }
 
-    const exportMessages = () => {
+    const exportMessages = async () => {
+        if (!storagePath && getStoragePathFromServer.data) {
+            storagePath = getStoragePathFromServer.data.storagePath
+        }
         const obj = {}
+        let fileSeparator = '/'
+        if ('windows' === getOS()) {
+            fileSeparator = '\\'
+        }
         for (let i = 0; i < allChatlogs.length; i += 1) {
             const chatmsg = allChatlogs[i]
             const chatPK = getChatPK(chatmsg)
+            let filePaths = []
+            if (chatmsg.fileUploads) {
+                chatmsg.fileUploads = JSON.parse(chatmsg.fileUploads)
+                chatmsg.fileUploads.forEach((file) => {
+                    if (file.type === 'stored-file') {
+                        filePaths.push(
+                            `${storagePath}${fileSeparator}${chatmsg.chatflowid}${fileSeparator}${chatmsg.chatId}${fileSeparator}${file.name}`
+                        )
+                    }
+                })
+            }
             const msg = {
                 content: chatmsg.content,
                 role: chatmsg.role === 'apiMessage' ? 'bot' : 'user',
                 time: chatmsg.createdDate
             }
+            if (filePaths.length) msg.filePaths = filePaths
             if (chatmsg.sourceDocuments) msg.sourceDocuments = JSON.parse(chatmsg.sourceDocuments)
             if (chatmsg.usedTools) msg.usedTools = JSON.parse(chatmsg.usedTools)
             if (chatmsg.fileAnnotations) msg.fileAnnotations = JSON.parse(chatmsg.fileAnnotations)
@@ -249,6 +278,14 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
                     })
                 }
             }
+            if (chatmsg.fileUploads) {
+                chatmsg.fileUploads = JSON.parse(chatmsg.fileUploads)
+                chatmsg.fileUploads.forEach((file) => {
+                    if (file.type === 'stored-file') {
+                        file.data = `${baseURL}/api/v1/get-upload-file?chatflowId=${chatmsg.chatflowid}&chatId=${chatmsg.chatId}&fileName=${file.name}`
+                    }
+                })
+            }
             const obj = {
                 ...chatmsg,
                 message: chatmsg.content,
@@ -357,6 +394,8 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
 
     useEffect(() => {
         if (getChatmessageApi.data) {
+            getStoragePathFromServer.request()
+
             setAllChatLogs(getChatmessageApi.data)
             const chatPK = processChatLogs(getChatmessageApi.data)
             setSelectedMessageIndex(0)
@@ -593,8 +632,8 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
                                                             sx={{
                                                                 background:
                                                                     message.type === 'apiMessage' ? theme.palette.asyncSelect.main : '',
-                                                                pl: 1,
-                                                                pr: 1
+                                                                py: '1rem',
+                                                                px: '1.5rem'
                                                             }}
                                                             key={index}
                                                             style={{ display: 'flex', justifyContent: 'center', alignContent: 'center' }}
@@ -644,6 +683,51 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
                                                                         })}
                                                                     </div>
                                                                 )}
+                                                                {message.fileUploads && message.fileUploads.length > 0 && (
+                                                                    <div
+                                                                        style={{
+                                                                            display: 'flex',
+                                                                            flexWrap: 'wrap',
+                                                                            flexDirection: 'column',
+                                                                            width: '100%',
+                                                                            gap: '8px'
+                                                                        }}
+                                                                    >
+                                                                        {message.fileUploads.map((item, index) => {
+                                                                            return (
+                                                                                <>
+                                                                                    {item.mime.startsWith('image/') ? (
+                                                                                        <Card
+                                                                                            key={index}
+                                                                                            sx={{
+                                                                                                p: 0,
+                                                                                                m: 0,
+                                                                                                maxWidth: 128,
+                                                                                                marginRight: '10px',
+                                                                                                flex: '0 0 auto'
+                                                                                            }}
+                                                                                        >
+                                                                                            <CardMedia
+                                                                                                component='img'
+                                                                                                image={item.data}
+                                                                                                sx={{ height: 64 }}
+                                                                                                alt={'preview'}
+                                                                                                style={messageImageStyle}
+                                                                                            />
+                                                                                        </Card>
+                                                                                    ) : (
+                                                                                        // eslint-disable-next-line jsx-a11y/media-has-caption
+                                                                                        <audio controls='controls'>
+                                                                                            Your browser does not support the &lt;audio&gt;
+                                                                                            tag.
+                                                                                            <source src={item.data} type={item.mime} />
+                                                                                        </audio>
+                                                                                    )}
+                                                                                </>
+                                                                            )
+                                                                        })}
+                                                                    </div>
+                                                                )}
                                                                 <div className='markdownanswer'>
                                                                     {/* Messages are being rendered in Markdown format */}
                                                                     <MemoizedReactMarkdown

packages/ui/src/utils/genericHelper.js

@@ -607,3 +607,25 @@ export const getConfigExamplesForCurl = (configData, bodyType, isMultiple, stopN
     }
     return finalStr
 }
+
+export const getOS = () => {
+    let userAgent = window.navigator.userAgent.toLowerCase(),
+        macosPlatforms = /(macintosh|macintel|macppc|mac68k|macos)/i,
+        windowsPlatforms = /(win32|win64|windows|wince)/i,
+        iosPlatforms = /(iphone|ipad|ipod)/i,
+        os = null
+
+    if (macosPlatforms.test(userAgent)) {
+        os = 'macos'
+    } else if (iosPlatforms.test(userAgent)) {
+        os = 'ios'
+    } else if (windowsPlatforms.test(userAgent)) {
+        os = 'windows'
+    } else if (/android/.test(userAgent)) {
+        os = 'android'
+    } else if (!os && /linux/.test(userAgent)) {
+        os = 'linux'
+    }
+
+    return os
+}

packages/ui/src/views/canvas/CanvasHeader.js

@@ -29,6 +29,7 @@ import useApi from 'hooks/useApi'
 import { generateExportFlowData } from 'utils/genericHelper'
 import { uiBaseURL } from 'store/constant'
 import { SET_CHATFLOW } from 'store/actions'
+import SpeechToTextDialog from '../../ui-component/dialog/SpeechToTextDialog'
 
 // ==============================|| CANVAS HEADER ||============================== //
 
@@ -47,6 +48,8 @@ const CanvasHeader = ({ chatflow, handleSaveFlow, handleDeleteFlow, handleLoadFl
     const [apiDialogProps, setAPIDialogProps] = useState({})
     const [analyseDialogOpen, setAnalyseDialogOpen] = useState(false)
     const [analyseDialogProps, setAnalyseDialogProps] = useState({})
+    const [speechToAudioDialogOpen, setSpeechToAudioDialogOpen] = useState(false)
+    const [speechToAudioDialogProps, setSpeechToAudioialogProps] = useState({})
     const [conversationStartersDialogOpen, setConversationStartersDialogOpen] = useState(false)
     const [conversationStartersDialogProps, setConversationStartersDialogProps] = useState({})
     const [viewMessagesDialogOpen, setViewMessagesDialogOpen] = useState(false)
@@ -80,6 +83,12 @@ const CanvasHeader = ({ chatflow, handleSaveFlow, handleDeleteFlow, handleLoadFl
                 chatflow: chatflow
             })
             setAnalyseDialogOpen(true)
+        } else if (setting === 'enableSpeechToText') {
+            setSpeechToAudioialogProps({
+                title: 'Speech to Text',
+                chatflow: chatflow
+            })
+            setSpeechToAudioDialogOpen(true)
         } else if (setting === 'viewMessages') {
             setViewMessagesDialogProps({
                 title: 'View Messages',
@@ -394,6 +403,11 @@ const CanvasHeader = ({ chatflow, handleSaveFlow, handleDeleteFlow, handleLoadFl
             />
             <APICodeDialog show={apiDialogOpen} dialogProps={apiDialogProps} onCancel={() => setAPIDialogOpen(false)} />
             <AnalyseFlowDialog show={analyseDialogOpen} dialogProps={analyseDialogProps} onCancel={() => setAnalyseDialogOpen(false)} />
+            <SpeechToTextDialog
+                show={speechToAudioDialogOpen}
+                dialogProps={speechToAudioDialogProps}
+                onCancel={() => setSpeechToAudioDialogOpen(false)}
+            />
             <StarterPromptsDialog
                 show={conversationStartersDialogOpen}
                 dialogProps={conversationStartersDialogProps}

packages/ui/src/views/canvas/CredentialInputHandler.js

@@ -1,5 +1,5 @@
 import PropTypes from 'prop-types'
-import { useRef, useState } from 'react'
+import { useEffect, useRef, useState } from 'react'
 
 // material-ui
 import { IconButton } from '@mui/material'
@@ -88,6 +88,10 @@ const CredentialInputHandler = ({ inputParam, data, onSelect, disabled = false }
         setShowSpecificCredentialDialog(true)
     }
 
+    useEffect(() => {
+        setCredentialId(data?.credential ?? '')
+    }, [data])
+
     return (
         <div ref={ref}>
             {inputParam && (

packages/ui/src/views/chatflows/index.js

@@ -47,6 +47,7 @@ const Chatflows = () => {
     const [view, setView] = React.useState(localStorage.getItem('flowDisplayStyle') || 'card')
 
     const handleChange = (event, nextView) => {
+        if (nextView === null) return
         localStorage.setItem('flowDisplayStyle', nextView)
         setView(nextView)
     }

packages/ui/src/views/chatmessage/ChatExpandDialog.js

@@ -7,7 +7,7 @@ import { ChatMessage } from './ChatMessage'
 import { StyledButton } from 'ui-component/button/StyledButton'
 import { IconEraser } from '@tabler/icons'
 
-const ChatExpandDialog = ({ show, dialogProps, onClear, onCancel }) => {
+const ChatExpandDialog = ({ show, dialogProps, onClear, onCancel, previews, setPreviews }) => {
     const portalElement = document.getElementById('portal')
     const customization = useSelector((state) => state.customization)
 
@@ -21,7 +21,7 @@ const ChatExpandDialog = ({ show, dialogProps, onClear, onCancel }) => {
             aria-describedby='alert-dialog-description'
             sx={{ overflow: 'visible' }}
         >
-            <DialogTitle sx={{ fontSize: '1rem' }} id='alert-dialog-title'>
+            <DialogTitle sx={{ fontSize: '1rem', p: 1.5 }} id='alert-dialog-title'>
                 <div style={{ display: 'flex', flexDirection: 'row' }}>
                     {dialogProps.title}
                     <div style={{ flex: 1 }}></div>
@@ -43,8 +43,17 @@ const ChatExpandDialog = ({ show, dialogProps, onClear, onCancel }) => {
                     )}
                 </div>
             </DialogTitle>
-            <DialogContent sx={{ display: 'flex', justifyContent: 'flex-end', flexDirection: 'column' }}>
-                <ChatMessage isDialog={true} open={dialogProps.open} chatflowid={dialogProps.chatflowid} />
+            <DialogContent
+                className='cloud-dialog-wrapper'
+                sx={{ display: 'flex', justifyContent: 'flex-end', flexDirection: 'column', p: 0 }}
+            >
+                <ChatMessage
+                    isDialog={true}
+                    open={dialogProps.open}
+                    chatflowid={dialogProps.chatflowid}
+                    previews={previews}
+                    setPreviews={setPreviews}
+                />
             </DialogContent>
         </Dialog>
     ) : null
@@ -56,7 +65,9 @@ ChatExpandDialog.propTypes = {
     show: PropTypes.bool,
     dialogProps: PropTypes.object,
     onClear: PropTypes.func,
-    onCancel: PropTypes.func
+    onCancel: PropTypes.func,
+    previews: PropTypes.array,
+    setPreviews: PropTypes.func
 }
 
 export default ChatExpandDialog