add upload files and tool features

packages/components/nodes/agents/OpenAIAssistant/OpenAIAssistant.ts

@@ -1,4 +1,4 @@
-import { ICommonObject, IDatabaseEntity, INode, INodeData, INodeOptionsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, IDatabaseEntity, INode, INodeData, INodeOptionsValue, INodeParams, IUsedTool } from '../../../src/Interface'
 import OpenAI from 'openai'
 import { DataSource } from 'typeorm'
 import { getCredentialData, getCredentialParam, getUserHome } from '../../../src/utils'
@@ -6,6 +6,8 @@ import { MessageContentImageFile, MessageContentText } from 'openai/resources/be
 import * as fsDefault from 'node:fs'
 import * as path from 'node:path'
 import fetch from 'node-fetch'
+import { flatten } from 'lodash'
+import { zodToJsonSchema } from 'zod-to-json-schema'
 
 class OpenAIAssistant_Agents implements INode {
     label: string
@@ -33,6 +35,12 @@ class OpenAIAssistant_Agents implements INode {
                 name: 'selectedAssistant',
                 type: 'asyncOptions',
                 loadMethod: 'listAssistants'
+            },
+            {
+                label: 'Allowed Tools',
+                name: 'tools',
+                type: 'Tool',
+                list: true
             }
         ]
     }
@@ -78,19 +86,28 @@ class OpenAIAssistant_Agents implements INode {
             id: selectedAssistantId
         })
 
-        if (!assistant) throw new Error(`Assistant ${selectedAssistantId} not found`)
+        if (!assistant) {
+            options.logger.error(`Assistant ${selectedAssistantId} not found`)
+            return
+        }
 
         if (!sessionId && options.chatId) {
             const chatmsg = await appDataSource.getRepository(databaseEntities['ChatMessage']).findOneBy({
                 chatId: options.chatId
             })
-            if (!chatmsg) throw new Error(`Chat Message with Chat Id: ${options.chatId} not found`)
+            if (!chatmsg) {
+                options.logger.error(`Chat Message with Chat Id: ${options.chatId} not found`)
+                return
+            }
             sessionId = chatmsg.sessionId
         }
 
         const credentialData = await getCredentialData(assistant.credential ?? '', options)
         const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData)
-        if (!openAIApiKey) throw new Error(`OpenAI ApiKey not found`)
+        if (!openAIApiKey) {
+            options.logger.error(`OpenAI ApiKey not found`)
+            return
+        }
 
         const openai = new OpenAI({ apiKey: openAIApiKey })
         options.logger.info(`Clearing OpenAI Thread ${sessionId}`)
@@ -102,6 +119,9 @@ class OpenAIAssistant_Agents implements INode {
         const selectedAssistantId = nodeData.inputs?.selectedAssistant as string
         const appDataSource = options.appDataSource as DataSource
         const databaseEntities = options.databaseEntities as IDatabaseEntity
+        let tools = nodeData.inputs?.tools
+        tools = flatten(tools)
+        const formattedTools = tools?.map((tool: any) => formatToOpenAIAssistantTool(tool)) ?? []
 
         const assistant = await appDataSource.getRepository(databaseEntities['Assistant']).findOneBy({
             id: selectedAssistantId
@@ -116,10 +136,15 @@ class OpenAIAssistant_Agents implements INode {
         const openai = new OpenAI({ apiKey: openAIApiKey })
 
         // Retrieve assistant
+        try {
             const assistantDetails = JSON.parse(assistant.details)
             const openAIAssistantId = assistantDetails.id
             const retrievedAssistant = await openai.beta.assistants.retrieve(openAIAssistantId)
 
+            if (formattedTools.length) {
+                await openai.beta.assistants.update(openAIAssistantId, { tools: formattedTools })
+            }
+
             const chatmessage = await appDataSource.getRepository(databaseEntities['ChatMessage']).findOneBy({
                 chatId: options.chatId
             })
@@ -144,6 +169,8 @@ class OpenAIAssistant_Agents implements INode {
                 assistant_id: retrievedAssistant.id
             })
 
+            const usedTools: IUsedTool[] = []
+
             const promise = (threadId: string, runId: string) => {
                 return new Promise((resolve, reject) => {
                     const timeout = setInterval(async () => {
@@ -151,7 +178,50 @@ class OpenAIAssistant_Agents implements INode {
                         const state = run.status
                         if (state === 'completed') {
                             clearInterval(timeout)
-                        resolve(run)
+                            resolve(state)
+                        } else if (state === 'requires_action') {
+                            if (run.required_action?.submit_tool_outputs.tool_calls) {
+                                clearInterval(timeout)
+                                const actions: ICommonObject[] = []
+                                run.required_action.submit_tool_outputs.tool_calls.forEach((item) => {
+                                    const functionCall = item.function
+                                    const args = JSON.parse(functionCall.arguments)
+                                    actions.push({
+                                        tool: functionCall.name,
+                                        toolInput: args,
+                                        toolCallId: item.id
+                                    })
+                                })
+
+                                const submitToolOutputs = []
+                                for (let i = 0; i < actions.length; i += 1) {
+                                    const tool = tools.find((tool: any) => tool.name === actions[i].tool)
+                                    if (!tool) continue
+                                    const toolOutput = await tool.call(actions[i].toolInput)
+                                    submitToolOutputs.push({
+                                        tool_call_id: actions[i].toolCallId,
+                                        output: toolOutput
+                                    })
+                                    usedTools.push({
+                                        tool: tool.name,
+                                        toolInput: actions[i].toolInput,
+                                        toolOutput
+                                    })
+                                }
+
+                                if (submitToolOutputs.length) {
+                                    await openai.beta.threads.runs.submitToolOutputs(threadId, runId, {
+                                        tool_outputs: submitToolOutputs
+                                    })
+                                    resolve(state)
+                                } else {
+                                    reject(
+                                        new Error(
+                                            `Error processing thread: ${state}, Thread ID: ${threadId}, Run ID: ${runId}. submit_tool_outputs.tool_calls are empty`
+                                        )
+                                    )
+                                }
+                            }
                         } else if (state === 'cancelled' || state === 'expired' || state === 'failed') {
                             clearInterval(timeout)
                             reject(new Error(`Error processing thread: ${state}, Thread ID: ${threadId}, Run ID: ${runId}`))
@@ -161,7 +231,10 @@ class OpenAIAssistant_Agents implements INode {
             }
 
             // Polling run status
-        await promise(threadId, runThread.id)
+            let state = await promise(threadId, runThread.id)
+            while (state === 'requires_action') {
+                state = await promise(threadId, runThread.id)
+            }
 
             // List messages
             const messages = await openai.beta.threads.messages.list(threadId)
@@ -192,7 +265,14 @@ class OpenAIAssistant_Agents implements INode {
                 }
             }
 
-        return { text: returnVal, assistant: { assistantId: openAIAssistantId, threadId, runId: runThread.id, messages: messageData } }
+            return {
+                text: returnVal,
+                usedTools,
+                assistant: { assistantId: openAIAssistantId, threadId, runId: runThread.id, messages: messageData }
+            }
+        } catch (error) {
+            throw new Error(error)
+        }
     }
 }
 
@@ -221,4 +301,15 @@ const downloadFile = async (fileObj: any, filePath: string, openAIApiKey: string
     }
 }
 
+const formatToOpenAIAssistantTool = (tool: any): OpenAI.Beta.AssistantCreateParams.AssistantToolsFunction => {
+    return {
+        type: 'function',
+        function: {
+            name: tool.name,
+            description: tool.description,
+            parameters: zodToJsonSchema(tool.schema)
+        }
+    }
+}
+
 module.exports = { nodeClass: OpenAIAssistant_Agents }

packages/components/package.json

@@ -72,7 +72,8 @@
         "srt-parser-2": "^1.2.3",
         "vm2": "^3.9.19",
         "weaviate-ts-client": "^1.1.0",
-        "ws": "^8.9.0"
+        "ws": "^8.9.0",
+        "zod-to-json-schema": "^3.21.4"
     },
     "devDependencies": {
         "@types/gulp": "4.0.9",

packages/components/src/Interface.ts

@@ -126,6 +126,12 @@ export interface IMessage {
     type: MessageType
 }
 
+export interface IUsedTool {
+    tool: string
+    toolInput: object
+    toolOutput: string | object
+}
+
 /**
  * Classes
  */

packages/server/package.json

@@ -54,8 +54,8 @@
         "express": "^4.17.3",
         "express-basic-auth": "^1.2.1",
         "express-rate-limit": "^6.9.0",
-        "flowise-components": "*",
+        "flowise-components": "1.4.0-rc.1",
-        "flowise-ui": "*",
+        "flowise-ui": "1.4.0-rc.1",
         "moment-timezone": "^0.5.34",
         "multer": "^1.4.5-lts.1",
         "mysql": "^2.18.1",

packages/server/src/Interface.ts

@@ -29,6 +29,7 @@ export interface IChatMessage {
     content: string
     chatflowid: string
     sourceDocuments?: string
+    usedTools?: string
     chatType: string
     chatId: string
     memoryType?: string

packages/server/src/commands/start.ts

@@ -69,6 +69,10 @@ export default class Start extends Command {
             logger.error('uncaughtException: ', err)
         })
 
+        process.on('unhandledRejection', (err) => {
+            logger.error('unhandledRejection: ', err)
+        })
+
         const { flags } = await this.parse(Start)
 
         if (flags.PORT) process.env.PORT = flags.PORT

packages/server/src/database/entities/ChatMessage.ts

@@ -20,6 +20,9 @@ export class ChatMessage implements IChatMessage {
     @Column({ nullable: true, type: 'text' })
     sourceDocuments?: string
 
+    @Column({ nullable: true, type: 'text' })
+    usedTools?: string
+
     @Column()
     chatType: string
 

packages/server/src/database/migrations/mysql/1699481607341-AddUsedToolsToChatMessage.ts

@@ -0,0 +1,12 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddUsedToolsToChatMessage1699481607341 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        const columnExists = await queryRunner.hasColumn('chat_message', 'usedTools')
+        if (!columnExists) queryRunner.query(`ALTER TABLE \`chat_message\` ADD COLUMN \`usedTools\` TEXT;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`chat_message\` DROP COLUMN \`usedTools\`;`)
+    }
+}

packages/server/src/database/migrations/mysql/index.ts

@@ -7,6 +7,7 @@ import { AddApiConfig1694099200729 } from './1694099200729-AddApiConfig'
 import { AddAnalytic1694432361423 } from './1694432361423-AddAnalytic'
 import { AddChatHistory1694658767766 } from './1694658767766-AddChatHistory'
 import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEntity'
+import { AddUsedToolsToChatMessage1699481607341 } from './1699481607341-AddUsedToolsToChatMessage'
 
 export const mysqlMigrations = [
     Init1693840429259,
@@ -17,5 +18,6 @@ export const mysqlMigrations = [
     AddApiConfig1694099200729,
     AddAnalytic1694432361423,
     AddChatHistory1694658767766,
-    AddAssistantEntity1699325775451
+    AddAssistantEntity1699325775451,
+    AddUsedToolsToChatMessage1699481607341
 ]

packages/server/src/database/migrations/postgres/1699481607341-AddUsedToolsToChatMessage.ts

@@ -0,0 +1,11 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddUsedToolsToChatMessage1699481607341 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "chat_message" ADD COLUMN IF NOT EXISTS "usedTools" TEXT;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "chat_flow" DROP COLUMN "usedTools";`)
+    }
+}

packages/server/src/database/migrations/postgres/index.ts

@@ -7,6 +7,7 @@ import { AddApiConfig1694099183389 } from './1694099183389-AddApiConfig'
 import { AddAnalytic1694432361423 } from './1694432361423-AddAnalytic'
 import { AddChatHistory1694658756136 } from './1694658756136-AddChatHistory'
 import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEntity'
+import { AddUsedToolsToChatMessage1699481607341 } from './1699481607341-AddUsedToolsToChatMessage'
 
 export const postgresMigrations = [
     Init1693891895163,
@@ -17,5 +18,6 @@ export const postgresMigrations = [
     AddApiConfig1694099183389,
     AddAnalytic1694432361423,
     AddChatHistory1694658756136,
-    AddAssistantEntity1699325775451
+    AddAssistantEntity1699325775451,
+    AddUsedToolsToChatMessage1699481607341
 ]

packages/server/src/database/migrations/sqlite/1699481607341-AddUsedToolsToChatMessage.ts

@@ -0,0 +1,20 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddUsedToolsToChatMessage1699481607341 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE "temp_chat_message" ("id" varchar PRIMARY KEY NOT NULL, "role" varchar NOT NULL, "chatflowid" varchar NOT NULL, "content" text NOT NULL, "sourceDocuments" text, "usedTools" text, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "chatType" VARCHAR NOT NULL DEFAULT 'INTERNAL', "chatId" VARCHAR NOT NULL, "memoryType" VARCHAR, "sessionId" VARCHAR);`
+        )
+        await queryRunner.query(
+            `INSERT INTO "temp_chat_message" ("id", "role", "chatflowid", "content", "sourceDocuments", "createdDate", "chatType", "chatId", "memoryType", "sessionId") SELECT "id", "role", "chatflowid", "content", "sourceDocuments", "createdDate", "chatType", "chatId", "memoryType", "sessionId" FROM "chat_message";`
+        )
+        await queryRunner.query(`DROP TABLE "chat_message";`)
+        await queryRunner.query(`ALTER TABLE "temp_chat_message" RENAME TO "chat_message";`)
+        await queryRunner.query(`CREATE INDEX "IDX_e574527322272fd838f4f0f3d3" ON "chat_message" ("chatflowid") ;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE IF EXISTS "temp_chat_message";`)
+        await queryRunner.query(`ALTER TABLE "chat_message" DROP COLUMN "usedTools";`)
+    }
+}

packages/server/src/database/migrations/sqlite/index.ts

@@ -7,6 +7,7 @@ import { AddApiConfig1694090982460 } from './1694090982460-AddApiConfig'
 import { AddAnalytic1694432361423 } from './1694432361423-AddAnalytic'
 import { AddChatHistory1694657778173 } from './1694657778173-AddChatHistory'
 import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEntity'
+import { AddUsedToolsToChatMessage1699481607341 } from './1699481607341-AddUsedToolsToChatMessage'
 
 export const sqliteMigrations = [
     Init1693835579790,
@@ -17,5 +18,6 @@ export const sqliteMigrations = [
     AddApiConfig1694090982460,
     AddAnalytic1694432361423,
     AddChatHistory1694657778173,
-    AddAssistantEntity1699325775451
+    AddAssistantEntity1699325775451,
+    AddUsedToolsToChatMessage1699481607341
 ]

packages/server/src/index.ts

@@ -49,7 +49,8 @@ import {
     replaceInputsWithConfig,
     getEncryptionKey,
     checkMemorySessionId,
-    clearSessionMemoryFromViewMessageDialog
+    clearSessionMemoryFromViewMessageDialog,
+    getUserHome
 } from './utils'
 import { cloneDeep, omit } from 'lodash'
 import { getDataSource } from './DataSource'
@@ -670,6 +671,15 @@ export class App {
             const openai = new OpenAI({ apiKey: openAIApiKey })
             const retrievedAssistant = await openai.beta.assistants.retrieve(req.params.id)
 
+            if (retrievedAssistant.file_ids && retrievedAssistant.file_ids.length) {
+                const files = []
+                for (const file_id of retrievedAssistant.file_ids) {
+                    const file = await openai.files.retrieve(file_id)
+                    files.push(file)
+                }
+                ;(retrievedAssistant as any).files = files
+            }
+
             return res.json(retrievedAssistant)
         })
 
@@ -701,7 +711,6 @@ export class App {
 
             const assistantDetails = JSON.parse(body.details)
 
-            if (!assistantDetails.id) {
             try {
                 const credential = await this.AppDataSource.getRepository(Credential).findOneBy({
                     id: body.credential
@@ -724,24 +733,66 @@ export class App {
                         })
                     }
                 }
+
+                if (assistantDetails.uploadFiles) {
+                    // Base64 strings
+                    let files: string[] = []
+                    const fileBase64 = assistantDetails.uploadFiles
+                    if (fileBase64.startsWith('[') && fileBase64.endsWith(']')) {
+                        files = JSON.parse(fileBase64)
+                    } else {
+                        files = [fileBase64]
+                    }
+
+                    const uploadedFiles = []
+                    for (const file of files) {
+                        const splitDataURI = file.split(',')
+                        const filename = splitDataURI.pop()?.split(':')[1] ?? ''
+                        const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
+                        const filePath = path.join(getUserHome(), '.flowise', 'openai-assistant', filename)
+                        if (!fs.existsSync(filePath)) fs.writeFileSync(filePath, bf)
+
+                        const createdFile = await openai.files.create({
+                            file: fs.createReadStream(filePath),
+                            purpose: 'assistants'
+                        })
+                        uploadedFiles.push(createdFile)
+
+                        fs.unlinkSync(filePath)
+                    }
+                    assistantDetails.files = [...assistantDetails.files, ...uploadedFiles]
+                }
+
+                if (!assistantDetails.id) {
                     const newAssistant = await openai.beta.assistants.create({
                         name: assistantDetails.name,
                         description: assistantDetails.description,
                         instructions: assistantDetails.instructions,
                         model: assistantDetails.model,
-                        tools
+                        tools,
+                        file_ids: (assistantDetails.files ?? []).map((file: OpenAI.Files.FileObject) => file.id)
                     })
+                    assistantDetails.id = newAssistant.id
+                } else {
+                    await openai.beta.assistants.update(assistantDetails.id, {
+                        name: assistantDetails.name,
+                        description: assistantDetails.description,
+                        instructions: assistantDetails.instructions,
+                        model: assistantDetails.model,
+                        tools,
+                        file_ids: (assistantDetails.files ?? []).map((file: OpenAI.Files.FileObject) => file.id)
+                    })
+                }
 
                 const newAssistantDetails = {
-                        ...assistantDetails,
+                    ...assistantDetails
-                        id: newAssistant.id
                 }
+                if (newAssistantDetails.uploadFiles) delete newAssistantDetails.uploadFiles
 
                 body.details = JSON.stringify(newAssistantDetails)
             } catch (error) {
                 return res.status(500).send(`Error creating new assistant: ${error}`)
             }
-            }
 
             const newAssistant = new Assistant()
             Object.assign(newAssistant, body)
@@ -790,18 +841,50 @@ export class App {
                         })
                     }
                 }
+
+                if (assistantDetails.uploadFiles) {
+                    // Base64 strings
+                    let files: string[] = []
+                    const fileBase64 = assistantDetails.uploadFiles
+                    if (fileBase64.startsWith('[') && fileBase64.endsWith(']')) {
+                        files = JSON.parse(fileBase64)
+                    } else {
+                        files = [fileBase64]
+                    }
+
+                    const uploadedFiles = []
+                    for (const file of files) {
+                        const splitDataURI = file.split(',')
+                        const filename = splitDataURI.pop()?.split(':')[1] ?? ''
+                        const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
+                        const filePath = path.join(getUserHome(), '.flowise', 'openai-assistant', filename)
+                        if (!fs.existsSync(filePath)) fs.writeFileSync(filePath, bf)
+
+                        const createdFile = await openai.files.create({
+                            file: fs.createReadStream(filePath),
+                            purpose: 'assistants'
+                        })
+                        uploadedFiles.push(createdFile)
+
+                        fs.unlinkSync(filePath)
+                    }
+                    assistantDetails.files = [...assistantDetails.files, ...uploadedFiles]
+                }
+
                 await openai.beta.assistants.update(openAIAssistantId, {
                     name: assistantDetails.name,
                     description: assistantDetails.description,
                     instructions: assistantDetails.instructions,
                     model: assistantDetails.model,
-                    tools
+                    tools,
+                    file_ids: (assistantDetails.files ?? []).map((file: OpenAI.Files.FileObject) => file.id)
                 })
 
                 const newAssistantDetails = {
                     ...assistantDetails,
                     id: openAIAssistantId
                 }
+                if (newAssistantDetails.uploadFiles) delete newAssistantDetails.uploadFiles
 
                 const updateAssistant = new Assistant()
                 body.details = JSON.stringify(newAssistantDetails)
@@ -828,14 +911,13 @@ export class App {
             }
 
             try {
-                const body = req.body
+                const assistantDetails = JSON.parse(assistant.details)
-                const assistantDetails = JSON.parse(body.details)
 
                 const credential = await this.AppDataSource.getRepository(Credential).findOneBy({
-                    id: body.credential
+                    id: assistant.credential
                 })
 
-                if (!credential) return res.status(404).send(`Credential ${body.credential} not found`)
+                if (!credential) return res.status(404).send(`Credential ${assistant.credential} not found`)
 
                 // Decrpyt credentialData
                 const decryptedCredentialData = await decryptCredentialData(credential.encryptedData)
@@ -844,11 +926,13 @@ export class App {
 
                 const openai = new OpenAI({ apiKey: openAIApiKey })
 
+                const results = await this.AppDataSource.getRepository(Assistant).delete({ id: req.params.id })
+
                 await openai.beta.assistants.del(assistantDetails.id)
 
-                const results = await this.AppDataSource.getRepository(Assistant).delete({ id: req.params.id })
                 return res.json(results)
-            } catch (error) {
+            } catch (error: any) {
+                if (error.status === 404 && error.type === 'invalid_request_error') return res.send('OK')
                 return res.status(500).send(`Error deleting assistant: ${error}`)
             }
         })
@@ -1389,6 +1473,7 @@ export class App {
                 sessionId
             }
             if (result?.sourceDocuments) apiMessage.sourceDocuments = JSON.stringify(result.sourceDocuments)
+            if (result?.usedTools) apiMessage.usedTools = JSON.stringify(result.usedTools)
             await this.addChatMessage(apiMessage)
 
             logger.debug(`[server]: Finished running ${nodeToExecuteData.label} (${nodeToExecuteData.id})`)

packages/ui/src/ui-component/dialog/SourceDocDialog.js

@@ -29,7 +29,7 @@ const SourceDocDialog = ({ show, dialogProps, onCancel }) => {
             aria-describedby='alert-dialog-description'
         >
             <DialogTitle sx={{ fontSize: '1rem' }} id='alert-dialog-title'>
-                Source Document
+                {dialogProps.title ?? 'Source Documents'}
             </DialogTitle>
             <DialogContent>
                 <ReactJson

packages/ui/src/ui-component/dialog/ViewMessagesDialog.js

@@ -129,6 +129,7 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
                 time: chatmsg.createdDate
             }
             if (chatmsg.sourceDocuments) msg.sourceDocuments = JSON.parse(chatmsg.sourceDocuments)
+            if (chatmsg.usedTools) msg.usedTools = JSON.parse(chatmsg.usedTools)
 
             if (!Object.prototype.hasOwnProperty.call(obj, chatPK)) {
                 obj[chatPK] = {
@@ -251,6 +252,8 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
                 type: chatmsg.role
             }
             if (chatmsg.sourceDocuments) obj.sourceDocuments = JSON.parse(chatmsg.sourceDocuments)
+            if (chatmsg.usedTools) obj.usedTools = JSON.parse(chatmsg.usedTools)
+
             loadedMessages.push(obj)
         }
         setChatMessages(loadedMessages)
@@ -315,8 +318,8 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
         window.open(data, '_blank')
     }
 
-    const onSourceDialogClick = (data) => {
+    const onSourceDialogClick = (data, title) => {
-        setSourceDialogProps({ data })
+        setSourceDialogProps({ data, title })
         setSourceDialogOpen(true)
     }
 
@@ -599,6 +602,24 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
                                                                     width: '100%'
                                                                 }}
                                                             >
+                                                                {message.usedTools && (
+                                                                    <div style={{ display: 'block', flexDirection: 'row', width: '100%' }}>
+                                                                        {message.usedTools.map((tool, index) => {
+                                                                            return (
+                                                                                <Chip
+                                                                                    size='small'
+                                                                                    key={index}
+                                                                                    label={tool.tool}
+                                                                                    component='a'
+                                                                                    sx={{ mr: 1, mt: 1 }}
+                                                                                    variant='outlined'
+                                                                                    clickable
+                                                                                    onClick={() => onSourceDialogClick(tool, 'Used Tools')}
+                                                                                />
+                                                                            )
+                                                                        })}
+                                                                    </div>
+                                                                )}
                                                                 <div className='markdownanswer'>
                                                                     {/* Messages are being rendered in Markdown format */}
                                                                     <MemoizedReactMarkdown

packages/ui/src/views/assistants/AssistantDialog.js

@@ -5,7 +5,7 @@ import { useDispatch } from 'react-redux'
 import { enqueueSnackbar as enqueueSnackbarAction, closeSnackbar as closeSnackbarAction } from 'store/actions'
 import { v4 as uuidv4 } from 'uuid'
 
-import { Box, Typography, Button, Dialog, DialogActions, DialogContent, DialogTitle, Stack, OutlinedInput } from '@mui/material'
+import { Box, Typography, Button, IconButton, Dialog, DialogActions, DialogContent, DialogTitle, Stack, OutlinedInput } from '@mui/material'
 
 import { StyledButton } from 'ui-component/button/StyledButton'
 import { TooltipWithParser } from 'ui-component/tooltip/TooltipWithParser'
@@ -13,6 +13,8 @@ import ConfirmDialog from 'ui-component/dialog/ConfirmDialog'
 import { Dropdown } from 'ui-component/dropdown/Dropdown'
 import { MultiDropdown } from 'ui-component/dropdown/MultiDropdown'
 import CredentialInputHandler from 'views/canvas/CredentialInputHandler'
+import { File } from 'ui-component/file/File'
+import { BackdropLoader } from 'ui-component/loading/BackdropLoader'
 
 // Icons
 import { IconX } from '@tabler/icons'
@@ -29,29 +31,21 @@ import useNotifier from 'utils/useNotifier'
 import { HIDE_CANVAS_DIALOG, SHOW_CANVAS_DIALOG } from 'store/actions'
 
 const assistantAvailableModels = [
-    {
-        label: 'gpt-4',
-        name: 'gpt-4'
-    },
     {
         label: 'gpt-4-1106-preview',
         name: 'gpt-4-1106-preview'
     },
-    {
-        label: 'gpt-4-vision-preview',
-        name: 'gpt-4-vision-preview'
-    },
     {
         label: 'gpt-4-0613',
         name: 'gpt-4-0613'
     },
     {
-        label: 'gpt-4-32k',
+        label: 'gpt-4-0314',
-        name: 'gpt-4-32k'
+        name: 'gpt-4-0314'
     },
     {
-        label: 'gpt-4-32k-0613',
+        label: 'gpt-4',
-        name: 'gpt-4-32k-0613'
+        name: 'gpt-4'
     },
     {
         label: 'gpt-3.5-turbo',
@@ -100,6 +94,9 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
     const [assistantCredential, setAssistantCredential] = useState('')
     const [assistantInstructions, setAssistantInstructions] = useState('')
     const [assistantTools, setAssistantTools] = useState(['code_interpreter', 'retrieval'])
+    const [assistantFiles, setAssistantFiles] = useState([])
+    const [uploadAssistantFiles, setUploadAssistantFiles] = useState('')
+    const [loading, setLoading] = useState(false)
 
     useEffect(() => {
         if (show) dispatch({ type: SHOW_CANVAS_DIALOG })
@@ -120,6 +117,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
             setAssistantModel(assistantDetails.model)
             setAssistantInstructions(assistantDetails.instructions)
             setAssistantTools(assistantDetails.tools ?? [])
+            setAssistantFiles(assistantDetails.files ?? [])
         }
     }, [getSpecificAssistantApi.data])
 
@@ -130,6 +128,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
             setAssistantDesc(getAssistantObjApi.data.description)
             setAssistantModel(getAssistantObjApi.data.model)
             setAssistantInstructions(getAssistantObjApi.data.instructions)
+            setAssistantFiles(getAssistantObjApi.data.files ?? [])
 
             let tools = []
             if (getAssistantObjApi.data.tools && getAssistantObjApi.data.tools.length) {
@@ -155,6 +154,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
             setAssistantModel(assistantDetails.model)
             setAssistantInstructions(assistantDetails.instructions)
             setAssistantTools(assistantDetails.tools ?? [])
+            setAssistantFiles(assistantDetails.files ?? [])
         } else if (dialogProps.type === 'EDIT' && dialogProps.assistantId) {
             // When assistant dialog is opened from OpenAIAssistant node in canvas
             getSpecificAssistantApi.request(dialogProps.assistantId)
@@ -177,6 +177,8 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
             setAssistantModel('')
             setAssistantInstructions('')
             setAssistantTools(['code_interpreter', 'retrieval'])
+            setUploadAssistantFiles('')
+            setAssistantFiles([])
         }
 
         return () => {
@@ -190,11 +192,15 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
             setAssistantModel('')
             setAssistantInstructions('')
             setAssistantTools(['code_interpreter', 'retrieval'])
+            setUploadAssistantFiles('')
+            setAssistantFiles([])
+            setLoading(false)
         }
         // eslint-disable-next-line react-hooks/exhaustive-deps
     }, [dialogProps])
 
     const addNewAssistant = async () => {
+        setLoading(true)
         try {
             const assistantDetails = {
                 id: openAIAssistantId,
@@ -202,7 +208,9 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
                 description: assistantDesc,
                 model: assistantModel,
                 instructions: assistantInstructions,
-                tools: assistantTools
+                tools: assistantTools,
+                files: assistantFiles,
+                uploadFiles: uploadAssistantFiles
             }
             const obj = {
                 details: JSON.stringify(assistantDetails),
@@ -226,6 +234,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
                 })
                 onConfirm(createResp.data.id)
             }
+            setLoading(false)
         } catch (error) {
             const errorData = error.response.data || `${error.response.status}: ${error.response.statusText}`
             enqueueSnackbar({
@@ -241,18 +250,22 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
                     )
                 }
             })
+            setLoading(false)
             onCancel()
         }
     }
 
     const saveAssistant = async () => {
+        setLoading(true)
         try {
             const assistantDetails = {
                 name: assistantName,
                 description: assistantDesc,
                 model: assistantModel,
                 instructions: assistantInstructions,
-                tools: assistantTools
+                tools: assistantTools,
+                files: assistantFiles,
+                uploadFiles: uploadAssistantFiles
             }
             const obj = {
                 details: JSON.stringify(assistantDetails),
@@ -275,6 +288,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
                 })
                 onConfirm(saveResp.data.id)
             }
+            setLoading(false)
         } catch (error) {
             const errorData = error.response.data || `${error.response.status}: ${error.response.statusText}`
             enqueueSnackbar({
@@ -290,6 +304,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
                     )
                 }
             })
+            setLoading(false)
             onCancel()
         }
     }
@@ -341,6 +356,10 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
         }
     }
 
+    const onFileDeleteClick = async (fileId) => {
+        setAssistantFiles(assistantFiles.filter((file) => file.id !== fileId))
+    }
+
     const component = show ? (
         <Dialog
             fullWidth
@@ -513,6 +532,49 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
                         value={assistantTools ?? 'choose an option'}
                     />
                 </Box>
+                <Box sx={{ p: 2 }}>
+                    <Stack sx={{ position: 'relative' }} direction='row'>
+                        <Typography variant='overline'>
+                            Knowledge Files
+                            <TooltipWithParser
+                                style={{ marginLeft: 10 }}
+                                title='Allow assistant to use the content from uploaded files for retrieval and code interpreter. MAX: 20 files'
+                            />
+                        </Typography>
+                    </Stack>
+                    <div style={{ display: 'flex', flexDirection: 'row' }}>
+                        {assistantFiles.map((file, index) => (
+                            <div
+                                key={index}
+                                style={{
+                                    display: 'flex',
+                                    flexDirection: 'row',
+                                    alignItems: 'center',
+                                    width: 'max-content',
+                                    height: 'max-content',
+                                    borderRadius: 15,
+                                    background: 'rgb(254,252,191)',
+                                    paddingLeft: 15,
+                                    paddingRight: 15,
+                                    paddingTop: 5,
+                                    paddingBottom: 5,
+                                    marginRight: 10
+                                }}
+                            >
+                                <span style={{ color: 'rgb(116,66,16)', marginRight: 10 }}>{file.filename}</span>
+                                <IconButton sx={{ height: 15, width: 15, p: 0 }} onClick={() => onFileDeleteClick(file.id)}>
+                                    <IconX />
+                                </IconButton>
+                            </div>
+                        ))}
+                    </div>
+                    <File
+                        key={uploadAssistantFiles}
+                        fileType='*'
+                        onChange={(newValue) => setUploadAssistantFiles(newValue)}
+                        value={uploadAssistantFiles ?? 'Choose a file to upload'}
+                    />
+                </Box>
             </DialogContent>
             <DialogActions>
                 {dialogProps.type === 'EDIT' && (
@@ -529,6 +591,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
                 </StyledButton>
             </DialogActions>
             <ConfirmDialog />
+            {loading && <BackdropLoader open={loading} />}
         </Dialog>
     ) : null
 

packages/ui/src/views/canvas/NodeInputHandler.js

@@ -22,6 +22,7 @@ import { isValidConnection } from 'utils/genericHelper'
 import { JsonEditorInput } from 'ui-component/json/JsonEditor'
 import { TooltipWithParser } from 'ui-component/tooltip/TooltipWithParser'
 import ToolDialog from 'views/tools/ToolDialog'
+import AssistantDialog from 'views/assistants/AssistantDialog'
 import FormatPromptValuesDialog from 'ui-component/dialog/FormatPromptValuesDialog'
 import CredentialInputHandler from './CredentialInputHandler'
 
@@ -31,7 +32,7 @@ import { getInputVariables } from 'utils/genericHelper'
 // const
 import { FLOWISE_CREDENTIAL_ID } from 'store/constant'
 
-const EDITABLE_TOOLS = ['selectedTool']
+const EDITABLE_OPTIONS = ['selectedTool', 'selectedAssistant']
 
 const CustomWidthTooltip = styled(({ className, ...props }) => <Tooltip {...props} classes={{ popper: className }} />)({
     [`& .${tooltipClasses.tooltip}`]: {
@@ -106,6 +107,14 @@ const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isA
                 confirmButtonName: 'Save',
                 toolId: inputValue
             })
+        } else if (inputParamName === 'selectedAssistant') {
+            setAsyncOptionEditDialogProps({
+                title: 'Edit Assistant',
+                type: 'EDIT',
+                cancelButtonName: 'Cancel',
+                confirmButtonName: 'Save',
+                assistantId: inputValue
+            })
         }
         setAsyncOptionEditDialog(inputParamName)
     }
@@ -118,6 +127,13 @@ const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isA
                 cancelButtonName: 'Cancel',
                 confirmButtonName: 'Add'
             })
+        } else if (inputParamName === 'selectedAssistant') {
+            setAsyncOptionEditDialogProps({
+                title: 'Add New Assistant',
+                type: 'ADD',
+                cancelButtonName: 'Cancel',
+                confirmButtonName: 'Add'
+            })
         }
         setAsyncOptionEditDialog(inputParamName)
     }
@@ -340,11 +356,11 @@ const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isA
                                         name={inputParam.name}
                                         nodeData={data}
                                         value={data.inputs[inputParam.name] ?? inputParam.default ?? 'choose an option'}
-                                        isCreateNewOption={EDITABLE_TOOLS.includes(inputParam.name)}
+                                        isCreateNewOption={EDITABLE_OPTIONS.includes(inputParam.name)}
                                         onSelect={(newValue) => (data.inputs[inputParam.name] = newValue)}
                                         onCreateNew={() => addAsyncOption(inputParam.name)}
                                     />
-                                    {EDITABLE_TOOLS.includes(inputParam.name) && data.inputs[inputParam.name] && (
+                                    {EDITABLE_OPTIONS.includes(inputParam.name) && data.inputs[inputParam.name] && (
                                         <IconButton
                                             title='Edit'
                                             color='primary'
@@ -361,11 +377,17 @@ const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isA
                 </>
             )}
             <ToolDialog
-                show={EDITABLE_TOOLS.includes(showAsyncOptionDialog)}
+                show={showAsyncOptionDialog === 'selectedTool'}
                 dialogProps={asyncOptionEditDialogProps}
                 onCancel={() => setAsyncOptionEditDialog('')}
                 onConfirm={onConfirmAsyncOption}
             ></ToolDialog>
+            <AssistantDialog
+                show={showAsyncOptionDialog === 'selectedAssistant'}
+                dialogProps={asyncOptionEditDialogProps}
+                onCancel={() => setAsyncOptionEditDialog('')}
+                onConfirm={onConfirmAsyncOption}
+            ></AssistantDialog>
         </div>
     )
 }

packages/ui/src/views/chatmessage/ChatMessage.js

@@ -57,8 +57,8 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
     const getChatmessageApi = useApi(chatmessageApi.getInternalChatmessageFromChatflow)
     const getIsChatflowStreamingApi = useApi(chatflowsApi.getIsChatflowStreaming)
 
-    const onSourceDialogClick = (data) => {
+    const onSourceDialogClick = (data, title) => {
-        setSourceDialogProps({ data })
+        setSourceDialogProps({ data, title })
         setSourceDialogOpen(true)
     }
 
@@ -139,7 +139,7 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
 
                     setMessages((prevMessages) => [
                         ...prevMessages,
-                        { message: text, sourceDocuments: data?.sourceDocuments, type: 'apiMessage' }
+                        { message: text, sourceDocuments: data?.sourceDocuments, usedTools: data?.usedTools, type: 'apiMessage' }
                     ])
                 }
 
@@ -182,6 +182,7 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
                     type: message.role
                 }
                 if (message.sourceDocuments) obj.sourceDocuments = JSON.parse(message.sourceDocuments)
+                if (message.usedTools) obj.usedTools = JSON.parse(message.usedTools)
                 return obj
             })
             setMessages((prevMessages) => [...prevMessages, ...loadedMessages])
@@ -284,6 +285,24 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
                                             <img src={userPNG} alt='Me' width='30' height='30' className='usericon' />
                                         )}
                                         <div style={{ display: 'flex', flexDirection: 'column', width: '100%' }}>
+                                            {message.usedTools && (
+                                                <div style={{ display: 'block', flexDirection: 'row', width: '100%' }}>
+                                                    {message.usedTools.map((tool, index) => {
+                                                        return (
+                                                            <Chip
+                                                                size='small'
+                                                                key={index}
+                                                                label={tool.tool}
+                                                                component='a'
+                                                                sx={{ mr: 1, mt: 1 }}
+                                                                variant='outlined'
+                                                                clickable
+                                                                onClick={() => onSourceDialogClick(tool, 'Used Tools')}
+                                                            />
+                                                        )
+                                                    })}
+                                                </div>
+                                            )}
                                             <div className='markdownanswer'>
                                                 {/* Messages are being rendered in Markdown format */}
                                                 <MemoizedReactMarkdown