Merge pull request #1200 from FlowiseAI/feature/OpenAI-Assistant

Feature/open ai assistant

packages/components/nodes/agents/OpenAIAssistant/OpenAIAssistant.ts

@@ -1,4 +1,4 @@
-import { ICommonObject, IDatabaseEntity, INode, INodeData, INodeOptionsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, IDatabaseEntity, INode, INodeData, INodeOptionsValue, INodeParams, IUsedTool } from '../../../src/Interface'
 import OpenAI from 'openai'
 import { DataSource } from 'typeorm'
 import { getCredentialData, getCredentialParam, getUserHome } from '../../../src/utils'
@@ -6,6 +6,8 @@ import { MessageContentImageFile, MessageContentText } from 'openai/resources/be
 import * as fsDefault from 'node:fs'
 import * as path from 'node:path'
 import fetch from 'node-fetch'
+import { flatten } from 'lodash'
+import { zodToJsonSchema } from 'zod-to-json-schema'
 
 class OpenAIAssistant_Agents implements INode {
     label: string
@@ -33,6 +35,12 @@ class OpenAIAssistant_Agents implements INode {
                 name: 'selectedAssistant',
                 type: 'asyncOptions',
                 loadMethod: 'listAssistants'
+            },
+            {
+                label: 'Allowed Tools',
+                name: 'tools',
+                type: 'Tool',
+                list: true
             }
         ]
     }
@@ -78,19 +86,28 @@ class OpenAIAssistant_Agents implements INode {
             id: selectedAssistantId
         })
 
-        if (!assistant) throw new Error(`Assistant ${selectedAssistantId} not found`)
+        if (!assistant) {
+            options.logger.error(`Assistant ${selectedAssistantId} not found`)
+            return
+        }
 
         if (!sessionId && options.chatId) {
             const chatmsg = await appDataSource.getRepository(databaseEntities['ChatMessage']).findOneBy({
                 chatId: options.chatId
             })
-            if (!chatmsg) throw new Error(`Chat Message with Chat Id: ${options.chatId} not found`)
+            if (!chatmsg) {
+                options.logger.error(`Chat Message with Chat Id: ${options.chatId} not found`)
+                return
+            }
             sessionId = chatmsg.sessionId
         }
 
         const credentialData = await getCredentialData(assistant.credential ?? '', options)
         const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData)
-        if (!openAIApiKey) throw new Error(`OpenAI ApiKey not found`)
+        if (!openAIApiKey) {
+            options.logger.error(`OpenAI ApiKey not found`)
+            return
+        }
 
         const openai = new OpenAI({ apiKey: openAIApiKey })
         options.logger.info(`Clearing OpenAI Thread ${sessionId}`)
@@ -102,6 +119,9 @@ class OpenAIAssistant_Agents implements INode {
         const selectedAssistantId = nodeData.inputs?.selectedAssistant as string
         const appDataSource = options.appDataSource as DataSource
         const databaseEntities = options.databaseEntities as IDatabaseEntity
+        let tools = nodeData.inputs?.tools
+        tools = flatten(tools)
+        const formattedTools = tools?.map((tool: any) => formatToOpenAIAssistantTool(tool)) ?? []
 
         const assistant = await appDataSource.getRepository(databaseEntities['Assistant']).findOneBy({
             id: selectedAssistantId
@@ -116,83 +136,143 @@ class OpenAIAssistant_Agents implements INode {
         const openai = new OpenAI({ apiKey: openAIApiKey })
 
         // Retrieve assistant
-        const assistantDetails = JSON.parse(assistant.details)
+        try {
-        const openAIAssistantId = assistantDetails.id
+            const assistantDetails = JSON.parse(assistant.details)
-        const retrievedAssistant = await openai.beta.assistants.retrieve(openAIAssistantId)
+            const openAIAssistantId = assistantDetails.id
+            const retrievedAssistant = await openai.beta.assistants.retrieve(openAIAssistantId)
 
-        const chatmessage = await appDataSource.getRepository(databaseEntities['ChatMessage']).findOneBy({
+            if (formattedTools.length) {
-            chatId: options.chatId
+                await openai.beta.assistants.update(openAIAssistantId, { tools: formattedTools })
-        })
-
-        let threadId = ''
-        if (!chatmessage) {
-            const thread = await openai.beta.threads.create({})
-            threadId = thread.id
-        } else {
-            const thread = await openai.beta.threads.retrieve(chatmessage.sessionId)
-            threadId = thread.id
-        }
-
-        // Add message to thread
-        await openai.beta.threads.messages.create(threadId, {
-            role: 'user',
-            content: input
-        })
-
-        // Run assistant thread
-        const runThread = await openai.beta.threads.runs.create(threadId, {
-            assistant_id: retrievedAssistant.id
-        })
-
-        const promise = (threadId: string, runId: string) => {
-            return new Promise((resolve, reject) => {
-                const timeout = setInterval(async () => {
-                    const run = await openai.beta.threads.runs.retrieve(threadId, runId)
-                    const state = run.status
-                    if (state === 'completed') {
-                        clearInterval(timeout)
-                        resolve(run)
-                    } else if (state === 'cancelled' || state === 'expired' || state === 'failed') {
-                        clearInterval(timeout)
-                        reject(new Error(`Error processing thread: ${state}, Thread ID: ${threadId}, Run ID: ${runId}`))
-                    }
-                }, 500)
-            })
-        }
-
-        // Polling run status
-        await promise(threadId, runThread.id)
-
-        // List messages
-        const messages = await openai.beta.threads.messages.list(threadId)
-        const messageData = messages.data ?? []
-        const assistantMessages = messageData.filter((msg) => msg.role === 'assistant')
-        if (!assistantMessages.length) return ''
-
-        let returnVal = ''
-        for (let i = 0; i < assistantMessages[0].content.length; i += 1) {
-            if (assistantMessages[0].content[i].type === 'text') {
-                const content = assistantMessages[0].content[i] as MessageContentText
-                returnVal += content.text.value
-
-                //TODO: handle annotations
-            } else {
-                const content = assistantMessages[0].content[i] as MessageContentImageFile
-                const fileId = content.image_file.file_id
-                const fileObj = await openai.files.retrieve(fileId)
-                const filePath = path.join(getUserHome(), '.flowise', 'openai-assistant', `${fileObj.filename}.png`)
-
-                await downloadFile(fileObj, filePath, openAIApiKey)
-
-                const bitmap = fsDefault.readFileSync(filePath)
-                const base64String = Buffer.from(bitmap).toString('base64')
-
-                const imgHTML = `<img src="data:image/png;base64,${base64String}" width="100%" height="max-content" alt="${fileObj.filename}" /><br/>`
-                returnVal += imgHTML
             }
-        }
 
-        return { text: returnVal, assistant: { assistantId: openAIAssistantId, threadId, runId: runThread.id, messages: messageData } }
+            const chatmessage = await appDataSource.getRepository(databaseEntities['ChatMessage']).findOneBy({
+                chatId: options.chatId
+            })
+
+            let threadId = ''
+            if (!chatmessage) {
+                const thread = await openai.beta.threads.create({})
+                threadId = thread.id
+            } else {
+                const thread = await openai.beta.threads.retrieve(chatmessage.sessionId)
+                threadId = thread.id
+            }
+
+            // Add message to thread
+            await openai.beta.threads.messages.create(threadId, {
+                role: 'user',
+                content: input
+            })
+
+            // Run assistant thread
+            const runThread = await openai.beta.threads.runs.create(threadId, {
+                assistant_id: retrievedAssistant.id
+            })
+
+            const usedTools: IUsedTool[] = []
+
+            const promise = (threadId: string, runId: string) => {
+                return new Promise((resolve, reject) => {
+                    const timeout = setInterval(async () => {
+                        const run = await openai.beta.threads.runs.retrieve(threadId, runId)
+                        const state = run.status
+                        if (state === 'completed') {
+                            clearInterval(timeout)
+                            resolve(state)
+                        } else if (state === 'requires_action') {
+                            if (run.required_action?.submit_tool_outputs.tool_calls) {
+                                clearInterval(timeout)
+                                const actions: ICommonObject[] = []
+                                run.required_action.submit_tool_outputs.tool_calls.forEach((item) => {
+                                    const functionCall = item.function
+                                    const args = JSON.parse(functionCall.arguments)
+                                    actions.push({
+                                        tool: functionCall.name,
+                                        toolInput: args,
+                                        toolCallId: item.id
+                                    })
+                                })
+
+                                const submitToolOutputs = []
+                                for (let i = 0; i < actions.length; i += 1) {
+                                    const tool = tools.find((tool: any) => tool.name === actions[i].tool)
+                                    if (!tool) continue
+                                    const toolOutput = await tool.call(actions[i].toolInput)
+                                    submitToolOutputs.push({
+                                        tool_call_id: actions[i].toolCallId,
+                                        output: toolOutput
+                                    })
+                                    usedTools.push({
+                                        tool: tool.name,
+                                        toolInput: actions[i].toolInput,
+                                        toolOutput
+                                    })
+                                }
+
+                                if (submitToolOutputs.length) {
+                                    await openai.beta.threads.runs.submitToolOutputs(threadId, runId, {
+                                        tool_outputs: submitToolOutputs
+                                    })
+                                    resolve(state)
+                                } else {
+                                    reject(
+                                        new Error(
+                                            `Error processing thread: ${state}, Thread ID: ${threadId}, Run ID: ${runId}. submit_tool_outputs.tool_calls are empty`
+                                        )
+                                    )
+                                }
+                            }
+                        } else if (state === 'cancelled' || state === 'expired' || state === 'failed') {
+                            clearInterval(timeout)
+                            reject(new Error(`Error processing thread: ${state}, Thread ID: ${threadId}, Run ID: ${runId}`))
+                        }
+                    }, 500)
+                })
+            }
+
+            // Polling run status
+            let state = await promise(threadId, runThread.id)
+            while (state === 'requires_action') {
+                state = await promise(threadId, runThread.id)
+            }
+
+            // List messages
+            const messages = await openai.beta.threads.messages.list(threadId)
+            const messageData = messages.data ?? []
+            const assistantMessages = messageData.filter((msg) => msg.role === 'assistant')
+            if (!assistantMessages.length) return ''
+
+            let returnVal = ''
+            for (let i = 0; i < assistantMessages[0].content.length; i += 1) {
+                if (assistantMessages[0].content[i].type === 'text') {
+                    const content = assistantMessages[0].content[i] as MessageContentText
+                    returnVal += content.text.value
+
+                    //TODO: handle annotations
+                } else {
+                    const content = assistantMessages[0].content[i] as MessageContentImageFile
+                    const fileId = content.image_file.file_id
+                    const fileObj = await openai.files.retrieve(fileId)
+                    const filePath = path.join(getUserHome(), '.flowise', 'openai-assistant', `${fileObj.filename}.png`)
+
+                    await downloadFile(fileObj, filePath, openAIApiKey)
+
+                    const bitmap = fsDefault.readFileSync(filePath)
+                    const base64String = Buffer.from(bitmap).toString('base64')
+
+                    const imgHTML = `<img src="data:image/png;base64,${base64String}" width="100%" height="max-content" alt="${fileObj.filename}" /><br/>`
+                    returnVal += imgHTML
+                }
+            }
+
+            return {
+                text: returnVal,
+                usedTools,
+                assistant: { assistantId: openAIAssistantId, threadId, runId: runThread.id, messages: messageData }
+            }
+        } catch (error) {
+            throw new Error(error)
+        }
     }
 }
 
@@ -221,4 +301,15 @@ const downloadFile = async (fileObj: any, filePath: string, openAIApiKey: string
     }
 }
 
+const formatToOpenAIAssistantTool = (tool: any): OpenAI.Beta.AssistantCreateParams.AssistantToolsFunction => {
+    return {
+        type: 'function',
+        function: {
+            name: tool.name,
+            description: tool.description,
+            parameters: zodToJsonSchema(tool.schema)
+        }
+    }
+}
+
 module.exports = { nodeClass: OpenAIAssistant_Agents }

packages/components/package.json

@@ -72,7 +72,8 @@
         "srt-parser-2": "^1.2.3",
         "vm2": "^3.9.19",
         "weaviate-ts-client": "^1.1.0",
-        "ws": "^8.9.0"
+        "ws": "^8.9.0",
+        "zod-to-json-schema": "^3.21.4"
     },
     "devDependencies": {
         "@types/gulp": "4.0.9",

packages/components/src/Interface.ts

@@ -126,6 +126,12 @@ export interface IMessage {
     type: MessageType
 }
 
+export interface IUsedTool {
+    tool: string
+    toolInput: object
+    toolOutput: string | object
+}
+
 /**
  * Classes
  */

packages/server/marketplaces/chatflows/Conversational Retrieval Agent.json

@@ -1,5 +1,6 @@
 {
     "description": "Agent optimized for vector retrieval during conversation and answering questions based on previous dialogue.",
+    "badge": "POPULAR",
     "nodes": [
         {
             "width": 300,

packages/server/marketplaces/chatflows/Conversational Retrieval QA Chain.json

@@ -1,5 +1,6 @@
 {
     "description": "Text file QnA using conversational retrieval QA chain",
+    "badge": "POPULAR",
     "nodes": [
         {
             "width": 300,

packages/server/marketplaces/chatflows/Flowise Docs QnA.json

@@ -1,5 +1,6 @@
 {
     "description": "Flowise Docs Github QnA using conversational retrieval QA chain",
+    "badge": "POPULAR",
     "nodes": [
         {
             "width": 300,

packages/server/marketplaces/chatflows/List Output Parser.json

@@ -1,5 +1,6 @@
 {
     "description": "Return response as a list (array) instead of a string/text",
+    "badge": "NEW",
     "nodes": [
         {
             "width": 300,

packages/server/marketplaces/chatflows/OpenAI Assistant.json

@@ -0,0 +1,234 @@
+{
+    "description": "OpenAI Assistant that has instructions and can leverage models, tools, and knowledge to respond to user queries",
+    "badge": "NEW",
+    "nodes": [
+        {
+            "width": 300,
+            "height": 327,
+            "id": "openAIAssistant_0",
+            "position": {
+                "x": 895.3722263184736,
+                "y": 118.50795801755544
+            },
+            "type": "customNode",
+            "data": {
+                "id": "openAIAssistant_0",
+                "label": "OpenAI Assistant",
+                "version": 1,
+                "name": "openAIAssistant",
+                "type": "OpenAIAssistant",
+                "baseClasses": ["OpenAIAssistant"],
+                "category": "Agents",
+                "description": "An agent that uses OpenAI Assistant API to pick the tool and args to call",
+                "inputParams": [
+                    {
+                        "label": "Select Assistant",
+                        "name": "selectedAssistant",
+                        "type": "asyncOptions",
+                        "loadMethod": "listAssistants",
+                        "id": "openAIAssistant_0-input-selectedAssistant-asyncOptions"
+                    }
+                ],
+                "inputAnchors": [
+                    {
+                        "label": "Allowed Tools",
+                        "name": "tools",
+                        "type": "Tool",
+                        "list": true,
+                        "id": "openAIAssistant_0-input-tools-Tool"
+                    }
+                ],
+                "inputs": {
+                    "selectedAssistant": "",
+                    "tools": ["{{calculator_0.data.instance}}", "{{serper_0.data.instance}}", "{{customTool_0.data.instance}}"]
+                },
+                "outputAnchors": [
+                    {
+                        "id": "openAIAssistant_0-output-openAIAssistant-OpenAIAssistant",
+                        "name": "openAIAssistant",
+                        "label": "OpenAIAssistant",
+                        "type": "OpenAIAssistant"
+                    }
+                ],
+                "outputs": {},
+                "selected": false
+            },
+            "selected": false,
+            "dragging": false,
+            "positionAbsolute": {
+                "x": 895.3722263184736,
+                "y": 118.50795801755544
+            }
+        },
+        {
+            "width": 300,
+            "height": 143,
+            "id": "calculator_0",
+            "position": {
+                "x": 454.74423492660145,
+                "y": -56.08375600705064
+            },
+            "type": "customNode",
+            "data": {
+                "id": "calculator_0",
+                "label": "Calculator",
+                "version": 1,
+                "name": "calculator",
+                "type": "Calculator",
+                "baseClasses": ["Calculator", "Tool", "StructuredTool", "Runnable"],
+                "category": "Tools",
+                "description": "Perform calculations on response",
+                "inputParams": [],
+                "inputAnchors": [],
+                "inputs": {},
+                "outputAnchors": [
+                    {
+                        "id": "calculator_0-output-calculator-Calculator|Tool|StructuredTool|Runnable",
+                        "name": "calculator",
+                        "label": "Calculator",
+                        "type": "Calculator | Tool | StructuredTool | Runnable"
+                    }
+                ],
+                "outputs": {},
+                "selected": false
+            },
+            "selected": false,
+            "positionAbsolute": {
+                "x": 454.74423492660145,
+                "y": -56.08375600705064
+            },
+            "dragging": false
+        },
+        {
+            "width": 300,
+            "height": 277,
+            "id": "customTool_0",
+            "position": {
+                "x": 454.43871855431365,
+                "y": 401.2171774551178
+            },
+            "type": "customNode",
+            "data": {
+                "id": "customTool_0",
+                "label": "Custom Tool",
+                "version": 1,
+                "name": "customTool",
+                "type": "CustomTool",
+                "baseClasses": ["CustomTool", "Tool", "StructuredTool", "Runnable"],
+                "category": "Tools",
+                "description": "Use custom tool you've created in Flowise within chatflow",
+                "inputParams": [
+                    {
+                        "label": "Select Tool",
+                        "name": "selectedTool",
+                        "type": "asyncOptions",
+                        "loadMethod": "listTools",
+                        "id": "customTool_0-input-selectedTool-asyncOptions"
+                    }
+                ],
+                "inputAnchors": [],
+                "inputs": {
+                    "selectedTool": ""
+                },
+                "outputAnchors": [
+                    {
+                        "id": "customTool_0-output-customTool-CustomTool|Tool|StructuredTool|Runnable",
+                        "name": "customTool",
+                        "label": "CustomTool",
+                        "type": "CustomTool | Tool | StructuredTool | Runnable"
+                    }
+                ],
+                "outputs": {},
+                "selected": false
+            },
+            "selected": false,
+            "positionAbsolute": {
+                "x": 454.43871855431365,
+                "y": 401.2171774551178
+            },
+            "dragging": false
+        },
+        {
+            "width": 300,
+            "height": 277,
+            "id": "serper_0",
+            "position": {
+                "x": 452.2514874331948,
+                "y": 99.6087116015905
+            },
+            "type": "customNode",
+            "data": {
+                "id": "serper_0",
+                "label": "Serper",
+                "version": 1,
+                "name": "serper",
+                "type": "Serper",
+                "baseClasses": ["Serper", "Tool", "StructuredTool", "Runnable"],
+                "category": "Tools",
+                "description": "Wrapper around Serper.dev - Google Search API",
+                "inputParams": [
+                    {
+                        "label": "Connect Credential",
+                        "name": "credential",
+                        "type": "credential",
+                        "credentialNames": ["serperApi"],
+                        "id": "serper_0-input-credential-credential"
+                    }
+                ],
+                "inputAnchors": [],
+                "inputs": {},
+                "outputAnchors": [
+                    {
+                        "id": "serper_0-output-serper-Serper|Tool|StructuredTool|Runnable",
+                        "name": "serper",
+                        "label": "Serper",
+                        "type": "Serper | Tool | StructuredTool | Runnable"
+                    }
+                ],
+                "outputs": {},
+                "selected": false
+            },
+            "selected": false,
+            "positionAbsolute": {
+                "x": 452.2514874331948,
+                "y": 99.6087116015905
+            },
+            "dragging": false
+        }
+    ],
+    "edges": [
+        {
+            "source": "calculator_0",
+            "sourceHandle": "calculator_0-output-calculator-Calculator|Tool|StructuredTool|Runnable",
+            "target": "openAIAssistant_0",
+            "targetHandle": "openAIAssistant_0-input-tools-Tool",
+            "type": "buttonedge",
+            "id": "calculator_0-calculator_0-output-calculator-Calculator|Tool|StructuredTool|Runnable-openAIAssistant_0-openAIAssistant_0-input-tools-Tool",
+            "data": {
+                "label": ""
+            }
+        },
+        {
+            "source": "serper_0",
+            "sourceHandle": "serper_0-output-serper-Serper|Tool|StructuredTool|Runnable",
+            "target": "openAIAssistant_0",
+            "targetHandle": "openAIAssistant_0-input-tools-Tool",
+            "type": "buttonedge",
+            "id": "serper_0-serper_0-output-serper-Serper|Tool|StructuredTool|Runnable-openAIAssistant_0-openAIAssistant_0-input-tools-Tool",
+            "data": {
+                "label": ""
+            }
+        },
+        {
+            "source": "customTool_0",
+            "sourceHandle": "customTool_0-output-customTool-CustomTool|Tool|StructuredTool|Runnable",
+            "target": "openAIAssistant_0",
+            "targetHandle": "openAIAssistant_0-input-tools-Tool",
+            "type": "buttonedge",
+            "id": "customTool_0-customTool_0-output-customTool-CustomTool|Tool|StructuredTool|Runnable-openAIAssistant_0-openAIAssistant_0-input-tools-Tool",
+            "data": {
+                "label": ""
+            }
+        }
+    ]
+}

packages/server/marketplaces/chatflows/Prompt Chaining with VectorStore.json

@@ -1,5 +1,6 @@
 {
     "description": "Use chat history to rephrase user question, and answer the rephrased question using retrieved docs from vector store",
+    "badge": "POPULAR",
     "nodes": [
         {
             "width": 300,

packages/server/marketplaces/chatflows/Simple Conversation Chain.json

@@ -1,5 +1,6 @@
 {
     "description": "Basic example of Conversation Chain with built-in memory - works exactly like ChatGPT",
+    "badge": "POPULAR",
     "nodes": [
         {
             "width": 300,

packages/server/marketplaces/chatflows/Structured Output Parser.json

@@ -1,5 +1,6 @@
 {
     "description": "Return response as a specified JSON structure instead of a string/text",
+    "badge": "NEW",
     "nodes": [
         {
             "width": 300,

packages/server/marketplaces/chatflows/WebPage QnA.json

@@ -1,5 +1,6 @@
 {
     "description": "Scrape web pages for QnA with long term memory Motorhead and return source documents",
+    "badge": "POPULAR",
     "nodes": [
         {
             "width": 300,

packages/server/package.json

@@ -54,8 +54,8 @@
         "express": "^4.17.3",
         "express-basic-auth": "^1.2.1",
         "express-rate-limit": "^6.9.0",
-        "flowise-components": "*",
+        "flowise-components": "1.4.0-rc.1",
-        "flowise-ui": "*",
+        "flowise-ui": "1.4.0-rc.1",
         "moment-timezone": "^0.5.34",
         "multer": "^1.4.5-lts.1",
         "mysql": "^2.18.1",

packages/server/src/Interface.ts

@@ -29,6 +29,7 @@ export interface IChatMessage {
     content: string
     chatflowid: string
     sourceDocuments?: string
+    usedTools?: string
     chatType: string
     chatId: string
     memoryType?: string

packages/server/src/commands/start.ts

@@ -69,6 +69,10 @@ export default class Start extends Command {
             logger.error('uncaughtException: ', err)
         })
 
+        process.on('unhandledRejection', (err) => {
+            logger.error('unhandledRejection: ', err)
+        })
+
         const { flags } = await this.parse(Start)
 
         if (flags.PORT) process.env.PORT = flags.PORT

packages/server/src/database/entities/ChatMessage.ts

@@ -20,6 +20,9 @@ export class ChatMessage implements IChatMessage {
     @Column({ nullable: true, type: 'text' })
     sourceDocuments?: string
 
+    @Column({ nullable: true, type: 'text' })
+    usedTools?: string
+
     @Column()
     chatType: string
 

packages/server/src/database/migrations/mysql/1699481607341-AddUsedToolsToChatMessage.ts

@@ -0,0 +1,12 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddUsedToolsToChatMessage1699481607341 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        const columnExists = await queryRunner.hasColumn('chat_message', 'usedTools')
+        if (!columnExists) queryRunner.query(`ALTER TABLE \`chat_message\` ADD COLUMN \`usedTools\` TEXT;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`chat_message\` DROP COLUMN \`usedTools\`;`)
+    }
+}

packages/server/src/database/migrations/mysql/index.ts

@@ -7,6 +7,7 @@ import { AddApiConfig1694099200729 } from './1694099200729-AddApiConfig'
 import { AddAnalytic1694432361423 } from './1694432361423-AddAnalytic'
 import { AddChatHistory1694658767766 } from './1694658767766-AddChatHistory'
 import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEntity'
+import { AddUsedToolsToChatMessage1699481607341 } from './1699481607341-AddUsedToolsToChatMessage'
 
 export const mysqlMigrations = [
     Init1693840429259,
@@ -17,5 +18,6 @@ export const mysqlMigrations = [
     AddApiConfig1694099200729,
     AddAnalytic1694432361423,
     AddChatHistory1694658767766,
-    AddAssistantEntity1699325775451
+    AddAssistantEntity1699325775451,
+    AddUsedToolsToChatMessage1699481607341
 ]

packages/server/src/database/migrations/postgres/1699481607341-AddUsedToolsToChatMessage.ts

@@ -0,0 +1,11 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddUsedToolsToChatMessage1699481607341 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "chat_message" ADD COLUMN IF NOT EXISTS "usedTools" TEXT;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "chat_flow" DROP COLUMN "usedTools";`)
+    }
+}

packages/server/src/database/migrations/postgres/index.ts

@@ -7,6 +7,7 @@ import { AddApiConfig1694099183389 } from './1694099183389-AddApiConfig'
 import { AddAnalytic1694432361423 } from './1694432361423-AddAnalytic'
 import { AddChatHistory1694658756136 } from './1694658756136-AddChatHistory'
 import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEntity'
+import { AddUsedToolsToChatMessage1699481607341 } from './1699481607341-AddUsedToolsToChatMessage'
 
 export const postgresMigrations = [
     Init1693891895163,
@@ -17,5 +18,6 @@ export const postgresMigrations = [
     AddApiConfig1694099183389,
     AddAnalytic1694432361423,
     AddChatHistory1694658756136,
-    AddAssistantEntity1699325775451
+    AddAssistantEntity1699325775451,
+    AddUsedToolsToChatMessage1699481607341
 ]

packages/server/src/database/migrations/sqlite/1699481607341-AddUsedToolsToChatMessage.ts

@@ -0,0 +1,20 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddUsedToolsToChatMessage1699481607341 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE "temp_chat_message" ("id" varchar PRIMARY KEY NOT NULL, "role" varchar NOT NULL, "chatflowid" varchar NOT NULL, "content" text NOT NULL, "sourceDocuments" text, "usedTools" text, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "chatType" VARCHAR NOT NULL DEFAULT 'INTERNAL', "chatId" VARCHAR NOT NULL, "memoryType" VARCHAR, "sessionId" VARCHAR);`
+        )
+        await queryRunner.query(
+            `INSERT INTO "temp_chat_message" ("id", "role", "chatflowid", "content", "sourceDocuments", "createdDate", "chatType", "chatId", "memoryType", "sessionId") SELECT "id", "role", "chatflowid", "content", "sourceDocuments", "createdDate", "chatType", "chatId", "memoryType", "sessionId" FROM "chat_message";`
+        )
+        await queryRunner.query(`DROP TABLE "chat_message";`)
+        await queryRunner.query(`ALTER TABLE "temp_chat_message" RENAME TO "chat_message";`)
+        await queryRunner.query(`CREATE INDEX "IDX_e574527322272fd838f4f0f3d3" ON "chat_message" ("chatflowid") ;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE IF EXISTS "temp_chat_message";`)
+        await queryRunner.query(`ALTER TABLE "chat_message" DROP COLUMN "usedTools";`)
+    }
+}

packages/server/src/database/migrations/sqlite/index.ts

@@ -7,6 +7,7 @@ import { AddApiConfig1694090982460 } from './1694090982460-AddApiConfig'
 import { AddAnalytic1694432361423 } from './1694432361423-AddAnalytic'
 import { AddChatHistory1694657778173 } from './1694657778173-AddChatHistory'
 import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEntity'
+import { AddUsedToolsToChatMessage1699481607341 } from './1699481607341-AddUsedToolsToChatMessage'
 
 export const sqliteMigrations = [
     Init1693835579790,
@@ -17,5 +18,6 @@ export const sqliteMigrations = [
     AddApiConfig1694090982460,
     AddAnalytic1694432361423,
     AddChatHistory1694657778173,
-    AddAssistantEntity1699325775451
+    AddAssistantEntity1699325775451,
+    AddUsedToolsToChatMessage1699481607341
 ]

packages/server/src/index.ts

@@ -49,7 +49,8 @@ import {
     replaceInputsWithConfig,
     getEncryptionKey,
     checkMemorySessionId,
-    clearSessionMemoryFromViewMessageDialog
+    clearSessionMemoryFromViewMessageDialog,
+    getUserHome
 } from './utils'
 import { cloneDeep, omit } from 'lodash'
 import { getDataSource } from './DataSource'
@@ -670,6 +671,15 @@ export class App {
             const openai = new OpenAI({ apiKey: openAIApiKey })
             const retrievedAssistant = await openai.beta.assistants.retrieve(req.params.id)
 
+            if (retrievedAssistant.file_ids && retrievedAssistant.file_ids.length) {
+                const files = []
+                for (const file_id of retrievedAssistant.file_ids) {
+                    const file = await openai.files.retrieve(file_id)
+                    files.push(file)
+                }
+                ;(retrievedAssistant as any).files = files
+            }
+
             return res.json(retrievedAssistant)
         })
 
@@ -701,46 +711,87 @@ export class App {
 
             const assistantDetails = JSON.parse(body.details)
 
-            if (!assistantDetails.id) {
+            try {
-                try {
+                const credential = await this.AppDataSource.getRepository(Credential).findOneBy({
-                    const credential = await this.AppDataSource.getRepository(Credential).findOneBy({
+                    id: body.credential
-                        id: body.credential
+                })
-                    })
 
-                    if (!credential) return res.status(404).send(`Credential ${body.credential} not found`)
+                if (!credential) return res.status(404).send(`Credential ${body.credential} not found`)
 
-                    // Decrpyt credentialData
+                // Decrpyt credentialData
-                    const decryptedCredentialData = await decryptCredentialData(credential.encryptedData)
+                const decryptedCredentialData = await decryptCredentialData(credential.encryptedData)
-                    const openAIApiKey = decryptedCredentialData['openAIApiKey']
+                const openAIApiKey = decryptedCredentialData['openAIApiKey']
-                    if (!openAIApiKey) return res.status(404).send(`OpenAI ApiKey not found`)
+                if (!openAIApiKey) return res.status(404).send(`OpenAI ApiKey not found`)
 
-                    const openai = new OpenAI({ apiKey: openAIApiKey })
+                const openai = new OpenAI({ apiKey: openAIApiKey })
 
-                    let tools = []
+                let tools = []
-                    if (assistantDetails.tools) {
+                if (assistantDetails.tools) {
-                        for (const tool of assistantDetails.tools ?? []) {
+                    for (const tool of assistantDetails.tools ?? []) {
-                            tools.push({
+                        tools.push({
-                                type: tool
+                            type: tool
-                            })
+                        })
-                        }
                     }
+                }
+
+                if (assistantDetails.uploadFiles) {
+                    // Base64 strings
+                    let files: string[] = []
+                    const fileBase64 = assistantDetails.uploadFiles
+                    if (fileBase64.startsWith('[') && fileBase64.endsWith(']')) {
+                        files = JSON.parse(fileBase64)
+                    } else {
+                        files = [fileBase64]
+                    }
+
+                    const uploadedFiles = []
+                    for (const file of files) {
+                        const splitDataURI = file.split(',')
+                        const filename = splitDataURI.pop()?.split(':')[1] ?? ''
+                        const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
+                        const filePath = path.join(getUserHome(), '.flowise', 'openai-assistant', filename)
+                        if (!fs.existsSync(filePath)) fs.writeFileSync(filePath, bf)
+
+                        const createdFile = await openai.files.create({
+                            file: fs.createReadStream(filePath),
+                            purpose: 'assistants'
+                        })
+                        uploadedFiles.push(createdFile)
+
+                        fs.unlinkSync(filePath)
+                    }
+                    assistantDetails.files = [...assistantDetails.files, ...uploadedFiles]
+                }
+
+                if (!assistantDetails.id) {
                     const newAssistant = await openai.beta.assistants.create({
                         name: assistantDetails.name,
                         description: assistantDetails.description,
                         instructions: assistantDetails.instructions,
                         model: assistantDetails.model,
-                        tools
+                        tools,
+                        file_ids: (assistantDetails.files ?? []).map((file: OpenAI.Files.FileObject) => file.id)
+                    })
+                    assistantDetails.id = newAssistant.id
+                } else {
+                    await openai.beta.assistants.update(assistantDetails.id, {
+                        name: assistantDetails.name,
+                        description: assistantDetails.description,
+                        instructions: assistantDetails.instructions,
+                        model: assistantDetails.model,
+                        tools,
+                        file_ids: (assistantDetails.files ?? []).map((file: OpenAI.Files.FileObject) => file.id)
                     })
-
-                    const newAssistantDetails = {
-                        ...assistantDetails,
-                        id: newAssistant.id
-                    }
-
-                    body.details = JSON.stringify(newAssistantDetails)
-                } catch (error) {
-                    return res.status(500).send(`Error creating new assistant: ${error}`)
                 }
+
+                const newAssistantDetails = {
+                    ...assistantDetails
+                }
+                if (newAssistantDetails.uploadFiles) delete newAssistantDetails.uploadFiles
+
+                body.details = JSON.stringify(newAssistantDetails)
+            } catch (error) {
+                return res.status(500).send(`Error creating new assistant: ${error}`)
             }
 
             const newAssistant = new Assistant()
@@ -790,18 +841,50 @@ export class App {
                         })
                     }
                 }
+
+                if (assistantDetails.uploadFiles) {
+                    // Base64 strings
+                    let files: string[] = []
+                    const fileBase64 = assistantDetails.uploadFiles
+                    if (fileBase64.startsWith('[') && fileBase64.endsWith(']')) {
+                        files = JSON.parse(fileBase64)
+                    } else {
+                        files = [fileBase64]
+                    }
+
+                    const uploadedFiles = []
+                    for (const file of files) {
+                        const splitDataURI = file.split(',')
+                        const filename = splitDataURI.pop()?.split(':')[1] ?? ''
+                        const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
+                        const filePath = path.join(getUserHome(), '.flowise', 'openai-assistant', filename)
+                        if (!fs.existsSync(filePath)) fs.writeFileSync(filePath, bf)
+
+                        const createdFile = await openai.files.create({
+                            file: fs.createReadStream(filePath),
+                            purpose: 'assistants'
+                        })
+                        uploadedFiles.push(createdFile)
+
+                        fs.unlinkSync(filePath)
+                    }
+                    assistantDetails.files = [...assistantDetails.files, ...uploadedFiles]
+                }
+
                 await openai.beta.assistants.update(openAIAssistantId, {
                     name: assistantDetails.name,
                     description: assistantDetails.description,
                     instructions: assistantDetails.instructions,
                     model: assistantDetails.model,
-                    tools
+                    tools,
+                    file_ids: (assistantDetails.files ?? []).map((file: OpenAI.Files.FileObject) => file.id)
                 })
 
                 const newAssistantDetails = {
                     ...assistantDetails,
                     id: openAIAssistantId
                 }
+                if (newAssistantDetails.uploadFiles) delete newAssistantDetails.uploadFiles
 
                 const updateAssistant = new Assistant()
                 body.details = JSON.stringify(newAssistantDetails)
@@ -828,14 +911,13 @@ export class App {
             }
 
             try {
-                const body = req.body
+                const assistantDetails = JSON.parse(assistant.details)
-                const assistantDetails = JSON.parse(body.details)
 
                 const credential = await this.AppDataSource.getRepository(Credential).findOneBy({
-                    id: body.credential
+                    id: assistant.credential
                 })
 
-                if (!credential) return res.status(404).send(`Credential ${body.credential} not found`)
+                if (!credential) return res.status(404).send(`Credential ${assistant.credential} not found`)
 
                 // Decrpyt credentialData
                 const decryptedCredentialData = await decryptCredentialData(credential.encryptedData)
@@ -844,11 +926,13 @@ export class App {
 
                 const openai = new OpenAI({ apiKey: openAIApiKey })
 
+                const results = await this.AppDataSource.getRepository(Assistant).delete({ id: req.params.id })
+
                 await openai.beta.assistants.del(assistantDetails.id)
 
-                const results = await this.AppDataSource.getRepository(Assistant).delete({ id: req.params.id })
                 return res.json(results)
-            } catch (error) {
+            } catch (error: any) {
+                if (error.status === 404 && error.type === 'invalid_request_error') return res.send('OK')
                 return res.status(500).send(`Error deleting assistant: ${error}`)
             }
         })
@@ -990,6 +1074,7 @@ export class App {
                     id: index,
                     name: file.split('.json')[0],
                     flowData: fileData.toString(),
+                    badge: fileDataObj?.badge,
                     description: fileDataObj?.description || ''
                 }
                 templates.push(template)
@@ -1389,6 +1474,7 @@ export class App {
                 sessionId
             }
             if (result?.sourceDocuments) apiMessage.sourceDocuments = JSON.stringify(result.sourceDocuments)
+            if (result?.usedTools) apiMessage.usedTools = JSON.stringify(result.usedTools)
             await this.addChatMessage(apiMessage)
 
             logger.debug(`[server]: Finished running ${nodeToExecuteData.label} (${nodeToExecuteData.id})`)

packages/ui/src/ui-component/dialog/SourceDocDialog.js

@@ -29,7 +29,7 @@ const SourceDocDialog = ({ show, dialogProps, onCancel }) => {
             aria-describedby='alert-dialog-description'
         >
             <DialogTitle sx={{ fontSize: '1rem' }} id='alert-dialog-title'>
-                Source Document
+                {dialogProps.title ?? 'Source Documents'}
             </DialogTitle>
             <DialogContent>
                 <ReactJson

packages/ui/src/ui-component/dialog/ViewMessagesDialog.js

@@ -129,6 +129,7 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
                 time: chatmsg.createdDate
             }
             if (chatmsg.sourceDocuments) msg.sourceDocuments = JSON.parse(chatmsg.sourceDocuments)
+            if (chatmsg.usedTools) msg.usedTools = JSON.parse(chatmsg.usedTools)
 
             if (!Object.prototype.hasOwnProperty.call(obj, chatPK)) {
                 obj[chatPK] = {
@@ -251,6 +252,8 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
                 type: chatmsg.role
             }
             if (chatmsg.sourceDocuments) obj.sourceDocuments = JSON.parse(chatmsg.sourceDocuments)
+            if (chatmsg.usedTools) obj.usedTools = JSON.parse(chatmsg.usedTools)
+
             loadedMessages.push(obj)
         }
         setChatMessages(loadedMessages)
@@ -315,8 +318,8 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
         window.open(data, '_blank')
     }
 
-    const onSourceDialogClick = (data) => {
+    const onSourceDialogClick = (data, title) => {
-        setSourceDialogProps({ data })
+        setSourceDialogProps({ data, title })
         setSourceDialogOpen(true)
     }
 
@@ -599,6 +602,24 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
                                                                     width: '100%'
                                                                 }}
                                                             >
+                                                                {message.usedTools && (
+                                                                    <div style={{ display: 'block', flexDirection: 'row', width: '100%' }}>
+                                                                        {message.usedTools.map((tool, index) => {
+                                                                            return (
+                                                                                <Chip
+                                                                                    size='small'
+                                                                                    key={index}
+                                                                                    label={tool.tool}
+                                                                                    component='a'
+                                                                                    sx={{ mr: 1, mt: 1 }}
+                                                                                    variant='outlined'
+                                                                                    clickable
+                                                                                    onClick={() => onSourceDialogClick(tool, 'Used Tools')}
+                                                                                />
+                                                                            )
+                                                                        })}
+                                                                    </div>
+                                                                )}
                                                                 <div className='markdownanswer'>
                                                                     {/* Messages are being rendered in Markdown format */}
                                                                     <MemoizedReactMarkdown

packages/ui/src/views/assistants/AssistantDialog.js

@@ -5,7 +5,7 @@ import { useDispatch } from 'react-redux'
 import { enqueueSnackbar as enqueueSnackbarAction, closeSnackbar as closeSnackbarAction } from 'store/actions'
 import { v4 as uuidv4 } from 'uuid'
 
-import { Box, Typography, Button, Dialog, DialogActions, DialogContent, DialogTitle, Stack, OutlinedInput } from '@mui/material'
+import { Box, Typography, Button, IconButton, Dialog, DialogActions, DialogContent, DialogTitle, Stack, OutlinedInput } from '@mui/material'
 
 import { StyledButton } from 'ui-component/button/StyledButton'
 import { TooltipWithParser } from 'ui-component/tooltip/TooltipWithParser'
@@ -13,6 +13,8 @@ import ConfirmDialog from 'ui-component/dialog/ConfirmDialog'
 import { Dropdown } from 'ui-component/dropdown/Dropdown'
 import { MultiDropdown } from 'ui-component/dropdown/MultiDropdown'
 import CredentialInputHandler from 'views/canvas/CredentialInputHandler'
+import { File } from 'ui-component/file/File'
+import { BackdropLoader } from 'ui-component/loading/BackdropLoader'
 
 // Icons
 import { IconX } from '@tabler/icons'
@@ -29,29 +31,21 @@ import useNotifier from 'utils/useNotifier'
 import { HIDE_CANVAS_DIALOG, SHOW_CANVAS_DIALOG } from 'store/actions'
 
 const assistantAvailableModels = [
-    {
-        label: 'gpt-4',
-        name: 'gpt-4'
-    },
     {
         label: 'gpt-4-1106-preview',
         name: 'gpt-4-1106-preview'
     },
-    {
-        label: 'gpt-4-vision-preview',
-        name: 'gpt-4-vision-preview'
-    },
     {
         label: 'gpt-4-0613',
         name: 'gpt-4-0613'
     },
     {
-        label: 'gpt-4-32k',
+        label: 'gpt-4-0314',
-        name: 'gpt-4-32k'
+        name: 'gpt-4-0314'
     },
     {
-        label: 'gpt-4-32k-0613',
+        label: 'gpt-4',
-        name: 'gpt-4-32k-0613'
+        name: 'gpt-4'
     },
     {
         label: 'gpt-3.5-turbo',
@@ -100,6 +94,9 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
     const [assistantCredential, setAssistantCredential] = useState('')
     const [assistantInstructions, setAssistantInstructions] = useState('')
     const [assistantTools, setAssistantTools] = useState(['code_interpreter', 'retrieval'])
+    const [assistantFiles, setAssistantFiles] = useState([])
+    const [uploadAssistantFiles, setUploadAssistantFiles] = useState('')
+    const [loading, setLoading] = useState(false)
 
     useEffect(() => {
         if (show) dispatch({ type: SHOW_CANVAS_DIALOG })
@@ -120,6 +117,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
             setAssistantModel(assistantDetails.model)
             setAssistantInstructions(assistantDetails.instructions)
             setAssistantTools(assistantDetails.tools ?? [])
+            setAssistantFiles(assistantDetails.files ?? [])
         }
     }, [getSpecificAssistantApi.data])
 
@@ -130,6 +128,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
             setAssistantDesc(getAssistantObjApi.data.description)
             setAssistantModel(getAssistantObjApi.data.model)
             setAssistantInstructions(getAssistantObjApi.data.instructions)
+            setAssistantFiles(getAssistantObjApi.data.files ?? [])
 
             let tools = []
             if (getAssistantObjApi.data.tools && getAssistantObjApi.data.tools.length) {
@@ -155,6 +154,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
             setAssistantModel(assistantDetails.model)
             setAssistantInstructions(assistantDetails.instructions)
             setAssistantTools(assistantDetails.tools ?? [])
+            setAssistantFiles(assistantDetails.files ?? [])
         } else if (dialogProps.type === 'EDIT' && dialogProps.assistantId) {
             // When assistant dialog is opened from OpenAIAssistant node in canvas
             getSpecificAssistantApi.request(dialogProps.assistantId)
@@ -177,6 +177,8 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
             setAssistantModel('')
             setAssistantInstructions('')
             setAssistantTools(['code_interpreter', 'retrieval'])
+            setUploadAssistantFiles('')
+            setAssistantFiles([])
         }
 
         return () => {
@@ -190,11 +192,15 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
             setAssistantModel('')
             setAssistantInstructions('')
             setAssistantTools(['code_interpreter', 'retrieval'])
+            setUploadAssistantFiles('')
+            setAssistantFiles([])
+            setLoading(false)
         }
         // eslint-disable-next-line react-hooks/exhaustive-deps
     }, [dialogProps])
 
     const addNewAssistant = async () => {
+        setLoading(true)
         try {
             const assistantDetails = {
                 id: openAIAssistantId,
@@ -202,7 +208,9 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
                 description: assistantDesc,
                 model: assistantModel,
                 instructions: assistantInstructions,
-                tools: assistantTools
+                tools: assistantTools,
+                files: assistantFiles,
+                uploadFiles: uploadAssistantFiles
             }
             const obj = {
                 details: JSON.stringify(assistantDetails),
@@ -226,6 +234,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
                 })
                 onConfirm(createResp.data.id)
             }
+            setLoading(false)
         } catch (error) {
             const errorData = error.response.data || `${error.response.status}: ${error.response.statusText}`
             enqueueSnackbar({
@@ -241,18 +250,22 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
                     )
                 }
             })
+            setLoading(false)
             onCancel()
         }
     }
 
     const saveAssistant = async () => {
+        setLoading(true)
         try {
             const assistantDetails = {
                 name: assistantName,
                 description: assistantDesc,
                 model: assistantModel,
                 instructions: assistantInstructions,
-                tools: assistantTools
+                tools: assistantTools,
+                files: assistantFiles,
+                uploadFiles: uploadAssistantFiles
             }
             const obj = {
                 details: JSON.stringify(assistantDetails),
@@ -275,6 +288,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
                 })
                 onConfirm(saveResp.data.id)
             }
+            setLoading(false)
         } catch (error) {
             const errorData = error.response.data || `${error.response.status}: ${error.response.statusText}`
             enqueueSnackbar({
@@ -290,6 +304,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
                     )
                 }
             })
+            setLoading(false)
             onCancel()
         }
     }
@@ -341,6 +356,10 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
         }
     }
 
+    const onFileDeleteClick = async (fileId) => {
+        setAssistantFiles(assistantFiles.filter((file) => file.id !== fileId))
+    }
+
     const component = show ? (
         <Dialog
             fullWidth
@@ -513,6 +532,49 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
                         value={assistantTools ?? 'choose an option'}
                     />
                 </Box>
+                <Box sx={{ p: 2 }}>
+                    <Stack sx={{ position: 'relative' }} direction='row'>
+                        <Typography variant='overline'>
+                            Knowledge Files
+                            <TooltipWithParser
+                                style={{ marginLeft: 10 }}
+                                title='Allow assistant to use the content from uploaded files for retrieval and code interpreter. MAX: 20 files'
+                            />
+                        </Typography>
+                    </Stack>
+                    <div style={{ display: 'flex', flexDirection: 'row' }}>
+                        {assistantFiles.map((file, index) => (
+                            <div
+                                key={index}
+                                style={{
+                                    display: 'flex',
+                                    flexDirection: 'row',
+                                    alignItems: 'center',
+                                    width: 'max-content',
+                                    height: 'max-content',
+                                    borderRadius: 15,
+                                    background: 'rgb(254,252,191)',
+                                    paddingLeft: 15,
+                                    paddingRight: 15,
+                                    paddingTop: 5,
+                                    paddingBottom: 5,
+                                    marginRight: 10
+                                }}
+                            >
+                                <span style={{ color: 'rgb(116,66,16)', marginRight: 10 }}>{file.filename}</span>
+                                <IconButton sx={{ height: 15, width: 15, p: 0 }} onClick={() => onFileDeleteClick(file.id)}>
+                                    <IconX />
+                                </IconButton>
+                            </div>
+                        ))}
+                    </div>
+                    <File
+                        key={uploadAssistantFiles}
+                        fileType='*'
+                        onChange={(newValue) => setUploadAssistantFiles(newValue)}
+                        value={uploadAssistantFiles ?? 'Choose a file to upload'}
+                    />
+                </Box>
             </DialogContent>
             <DialogActions>
                 {dialogProps.type === 'EDIT' && (
@@ -529,6 +591,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
                 </StyledButton>
             </DialogActions>
             <ConfirmDialog />
+            {loading && <BackdropLoader open={loading} />}
         </Dialog>
     ) : null
 

packages/ui/src/views/canvas/NodeInputHandler.js

@@ -22,6 +22,7 @@ import { isValidConnection } from 'utils/genericHelper'
 import { JsonEditorInput } from 'ui-component/json/JsonEditor'
 import { TooltipWithParser } from 'ui-component/tooltip/TooltipWithParser'
 import ToolDialog from 'views/tools/ToolDialog'
+import AssistantDialog from 'views/assistants/AssistantDialog'
 import FormatPromptValuesDialog from 'ui-component/dialog/FormatPromptValuesDialog'
 import CredentialInputHandler from './CredentialInputHandler'
 
@@ -31,7 +32,7 @@ import { getInputVariables } from 'utils/genericHelper'
 // const
 import { FLOWISE_CREDENTIAL_ID } from 'store/constant'
 
-const EDITABLE_TOOLS = ['selectedTool']
+const EDITABLE_OPTIONS = ['selectedTool', 'selectedAssistant']
 
 const CustomWidthTooltip = styled(({ className, ...props }) => <Tooltip {...props} classes={{ popper: className }} />)({
     [`& .${tooltipClasses.tooltip}`]: {
@@ -106,6 +107,14 @@ const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isA
                 confirmButtonName: 'Save',
                 toolId: inputValue
             })
+        } else if (inputParamName === 'selectedAssistant') {
+            setAsyncOptionEditDialogProps({
+                title: 'Edit Assistant',
+                type: 'EDIT',
+                cancelButtonName: 'Cancel',
+                confirmButtonName: 'Save',
+                assistantId: inputValue
+            })
         }
         setAsyncOptionEditDialog(inputParamName)
     }
@@ -118,6 +127,13 @@ const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isA
                 cancelButtonName: 'Cancel',
                 confirmButtonName: 'Add'
             })
+        } else if (inputParamName === 'selectedAssistant') {
+            setAsyncOptionEditDialogProps({
+                title: 'Add New Assistant',
+                type: 'ADD',
+                cancelButtonName: 'Cancel',
+                confirmButtonName: 'Add'
+            })
         }
         setAsyncOptionEditDialog(inputParamName)
     }
@@ -340,11 +356,11 @@ const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isA
                                         name={inputParam.name}
                                         nodeData={data}
                                         value={data.inputs[inputParam.name] ?? inputParam.default ?? 'choose an option'}
-                                        isCreateNewOption={EDITABLE_TOOLS.includes(inputParam.name)}
+                                        isCreateNewOption={EDITABLE_OPTIONS.includes(inputParam.name)}
                                         onSelect={(newValue) => (data.inputs[inputParam.name] = newValue)}
                                         onCreateNew={() => addAsyncOption(inputParam.name)}
                                     />
-                                    {EDITABLE_TOOLS.includes(inputParam.name) && data.inputs[inputParam.name] && (
+                                    {EDITABLE_OPTIONS.includes(inputParam.name) && data.inputs[inputParam.name] && (
                                         <IconButton
                                             title='Edit'
                                             color='primary'
@@ -361,11 +377,17 @@ const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isA
                 </>
             )}
             <ToolDialog
-                show={EDITABLE_TOOLS.includes(showAsyncOptionDialog)}
+                show={showAsyncOptionDialog === 'selectedTool'}
                 dialogProps={asyncOptionEditDialogProps}
                 onCancel={() => setAsyncOptionEditDialog('')}
                 onConfirm={onConfirmAsyncOption}
             ></ToolDialog>
+            <AssistantDialog
+                show={showAsyncOptionDialog === 'selectedAssistant'}
+                dialogProps={asyncOptionEditDialogProps}
+                onCancel={() => setAsyncOptionEditDialog('')}
+                onConfirm={onConfirmAsyncOption}
+            ></AssistantDialog>
         </div>
     )
 }

packages/ui/src/views/chatmessage/ChatMessage.js

@@ -57,8 +57,8 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
     const getChatmessageApi = useApi(chatmessageApi.getInternalChatmessageFromChatflow)
     const getIsChatflowStreamingApi = useApi(chatflowsApi.getIsChatflowStreaming)
 
-    const onSourceDialogClick = (data) => {
+    const onSourceDialogClick = (data, title) => {
-        setSourceDialogProps({ data })
+        setSourceDialogProps({ data, title })
         setSourceDialogOpen(true)
     }
 
@@ -139,7 +139,7 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
 
                     setMessages((prevMessages) => [
                         ...prevMessages,
-                        { message: text, sourceDocuments: data?.sourceDocuments, type: 'apiMessage' }
+                        { message: text, sourceDocuments: data?.sourceDocuments, usedTools: data?.usedTools, type: 'apiMessage' }
                     ])
                 }
 
@@ -182,6 +182,7 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
                     type: message.role
                 }
                 if (message.sourceDocuments) obj.sourceDocuments = JSON.parse(message.sourceDocuments)
+                if (message.usedTools) obj.usedTools = JSON.parse(message.usedTools)
                 return obj
             })
             setMessages((prevMessages) => [...prevMessages, ...loadedMessages])
@@ -284,6 +285,24 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
                                             <img src={userPNG} alt='Me' width='30' height='30' className='usericon' />
                                         )}
                                         <div style={{ display: 'flex', flexDirection: 'column', width: '100%' }}>
+                                            {message.usedTools && (
+                                                <div style={{ display: 'block', flexDirection: 'row', width: '100%' }}>
+                                                    {message.usedTools.map((tool, index) => {
+                                                        return (
+                                                            <Chip
+                                                                size='small'
+                                                                key={index}
+                                                                label={tool.tool}
+                                                                component='a'
+                                                                sx={{ mr: 1, mt: 1 }}
+                                                                variant='outlined'
+                                                                clickable
+                                                                onClick={() => onSourceDialogClick(tool, 'Used Tools')}
+                                                            />
+                                                        )
+                                                    })}
+                                                </div>
+                                            )}
                                             <div className='markdownanswer'>
                                                 {/* Messages are being rendered in Markdown format */}
                                                 <MemoizedReactMarkdown

packages/ui/src/views/marketplaces/index.js

@@ -4,7 +4,7 @@ import { useSelector } from 'react-redux'
 import PropTypes from 'prop-types'
 
 // material-ui
-import { Grid, Box, Stack, Tabs, Tab } from '@mui/material'
+import { Grid, Box, Stack, Tabs, Tab, Badge } from '@mui/material'
 import { useTheme } from '@mui/material/styles'
 import { IconHierarchy, IconTool } from '@tabler/icons'
 
@@ -157,7 +157,22 @@ const Marketplace = () => {
                                     getAllChatflowsMarketplacesApi.data &&
                                     getAllChatflowsMarketplacesApi.data.map((data, index) => (
                                         <Grid key={index} item lg={3} md={4} sm={6} xs={12}>
-                                            <ItemCard onClick={() => goToCanvas(data)} data={data} images={images[data.id]} />
+                                            {data.badge && (
+                                                <Badge
+                                                    sx={{
+                                                        '& .MuiBadge-badge': {
+                                                            right: 20
+                                                        }
+                                                    }}
+                                                    badgeContent={data.badge}
+                                                    color={data.badge === 'POPULAR' ? 'primary' : 'error'}
+                                                >
+                                                    <ItemCard onClick={() => goToCanvas(data)} data={data} images={images[data.id]} />
+                                                </Badge>
+                                            )}
+                                            {!data.badge && (
+                                                <ItemCard onClick={() => goToCanvas(data)} data={data} images={images[data.id]} />
+                                            )}
                                         </Grid>
                                     ))}
                             </Grid>
@@ -168,7 +183,20 @@ const Marketplace = () => {
                                     getAllToolsMarketplacesApi.data &&
                                     getAllToolsMarketplacesApi.data.map((data, index) => (
                                         <Grid key={index} item lg={3} md={4} sm={6} xs={12}>
-                                            <ItemCard data={data} onClick={() => goToTool(data)} />
+                                            {data.badge && (
+                                                <Badge
+                                                    sx={{
+                                                        '& .MuiBadge-badge': {
+                                                            right: 20
+                                                        }
+                                                    }}
+                                                    badgeContent={data.badge}
+                                                    color={data.badge === 'POPULAR' ? 'primary' : 'error'}
+                                                >
+                                                    <ItemCard data={data} onClick={() => goToTool(data)} />
+                                                </Badge>
+                                            )}
+                                            {!data.badge && <ItemCard data={data} onClick={() => goToTool(data)} />}
                                         </Grid>
                                     ))}
                             </Grid>