diff --git a/packages/components/nodes/agents/OpenAIAssistant/OpenAIAssistant.ts b/packages/components/nodes/agents/OpenAIAssistant/OpenAIAssistant.ts index 03a52560..21b2ce6b 100644 --- a/packages/components/nodes/agents/OpenAIAssistant/OpenAIAssistant.ts +++ b/packages/components/nodes/agents/OpenAIAssistant/OpenAIAssistant.ts @@ -1,4 +1,4 @@ -import { ICommonObject, IDatabaseEntity, INode, INodeData, INodeOptionsValue, INodeParams } from '../../../src/Interface' +import { ICommonObject, IDatabaseEntity, INode, INodeData, INodeOptionsValue, INodeParams, IUsedTool } from '../../../src/Interface' import OpenAI from 'openai' import { DataSource } from 'typeorm' import { getCredentialData, getCredentialParam, getUserHome } from '../../../src/utils' @@ -6,6 +6,8 @@ import { MessageContentImageFile, MessageContentText } from 'openai/resources/be import * as fsDefault from 'node:fs' import * as path from 'node:path' import fetch from 'node-fetch' +import { flatten } from 'lodash' +import { zodToJsonSchema } from 'zod-to-json-schema' class OpenAIAssistant_Agents implements INode { label: string @@ -33,6 +35,12 @@ class OpenAIAssistant_Agents implements INode { name: 'selectedAssistant', type: 'asyncOptions', loadMethod: 'listAssistants' + }, + { + label: 'Allowed Tools', + name: 'tools', + type: 'Tool', + list: true } ] } @@ -78,19 +86,28 @@ class OpenAIAssistant_Agents implements INode { id: selectedAssistantId }) - if (!assistant) throw new Error(`Assistant ${selectedAssistantId} not found`) + if (!assistant) { + options.logger.error(`Assistant ${selectedAssistantId} not found`) + return + } if (!sessionId && options.chatId) { const chatmsg = await appDataSource.getRepository(databaseEntities['ChatMessage']).findOneBy({ chatId: options.chatId }) - if (!chatmsg) throw new Error(`Chat Message with Chat Id: ${options.chatId} not found`) + if (!chatmsg) { + options.logger.error(`Chat Message with Chat Id: ${options.chatId} not found`) + return + } sessionId = chatmsg.sessionId } const credentialData = await getCredentialData(assistant.credential ?? '', options) const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData) - if (!openAIApiKey) throw new Error(`OpenAI ApiKey not found`) + if (!openAIApiKey) { + options.logger.error(`OpenAI ApiKey not found`) + return + } const openai = new OpenAI({ apiKey: openAIApiKey }) options.logger.info(`Clearing OpenAI Thread ${sessionId}`) @@ -102,6 +119,9 @@ class OpenAIAssistant_Agents implements INode { const selectedAssistantId = nodeData.inputs?.selectedAssistant as string const appDataSource = options.appDataSource as DataSource const databaseEntities = options.databaseEntities as IDatabaseEntity + let tools = nodeData.inputs?.tools + tools = flatten(tools) + const formattedTools = tools?.map((tool: any) => formatToOpenAIAssistantTool(tool)) ?? [] const assistant = await appDataSource.getRepository(databaseEntities['Assistant']).findOneBy({ id: selectedAssistantId @@ -116,83 +136,143 @@ class OpenAIAssistant_Agents implements INode { const openai = new OpenAI({ apiKey: openAIApiKey }) // Retrieve assistant - const assistantDetails = JSON.parse(assistant.details) - const openAIAssistantId = assistantDetails.id - const retrievedAssistant = await openai.beta.assistants.retrieve(openAIAssistantId) + try { + const assistantDetails = JSON.parse(assistant.details) + const openAIAssistantId = assistantDetails.id + const retrievedAssistant = await openai.beta.assistants.retrieve(openAIAssistantId) - const chatmessage = await appDataSource.getRepository(databaseEntities['ChatMessage']).findOneBy({ - chatId: options.chatId - }) - - let threadId = '' - if (!chatmessage) { - const thread = await openai.beta.threads.create({}) - threadId = thread.id - } else { - const thread = await openai.beta.threads.retrieve(chatmessage.sessionId) - threadId = thread.id - } - - // Add message to thread - await openai.beta.threads.messages.create(threadId, { - role: 'user', - content: input - }) - - // Run assistant thread - const runThread = await openai.beta.threads.runs.create(threadId, { - assistant_id: retrievedAssistant.id - }) - - const promise = (threadId: string, runId: string) => { - return new Promise((resolve, reject) => { - const timeout = setInterval(async () => { - const run = await openai.beta.threads.runs.retrieve(threadId, runId) - const state = run.status - if (state === 'completed') { - clearInterval(timeout) - resolve(run) - } else if (state === 'cancelled' || state === 'expired' || state === 'failed') { - clearInterval(timeout) - reject(new Error(`Error processing thread: ${state}, Thread ID: ${threadId}, Run ID: ${runId}`)) - } - }, 500) - }) - } - - // Polling run status - await promise(threadId, runThread.id) - - // List messages - const messages = await openai.beta.threads.messages.list(threadId) - const messageData = messages.data ?? [] - const assistantMessages = messageData.filter((msg) => msg.role === 'assistant') - if (!assistantMessages.length) return '' - - let returnVal = '' - for (let i = 0; i < assistantMessages[0].content.length; i += 1) { - if (assistantMessages[0].content[i].type === 'text') { - const content = assistantMessages[0].content[i] as MessageContentText - returnVal += content.text.value - - //TODO: handle annotations - } else { - const content = assistantMessages[0].content[i] as MessageContentImageFile - const fileId = content.image_file.file_id - const fileObj = await openai.files.retrieve(fileId) - const filePath = path.join(getUserHome(), '.flowise', 'openai-assistant', `${fileObj.filename}.png`) - - await downloadFile(fileObj, filePath, openAIApiKey) - - const bitmap = fsDefault.readFileSync(filePath) - const base64String = Buffer.from(bitmap).toString('base64') - - const imgHTML = `${fileObj.filename}
` - returnVal += imgHTML + if (formattedTools.length) { + await openai.beta.assistants.update(openAIAssistantId, { tools: formattedTools }) } - } - return { text: returnVal, assistant: { assistantId: openAIAssistantId, threadId, runId: runThread.id, messages: messageData } } + const chatmessage = await appDataSource.getRepository(databaseEntities['ChatMessage']).findOneBy({ + chatId: options.chatId + }) + + let threadId = '' + if (!chatmessage) { + const thread = await openai.beta.threads.create({}) + threadId = thread.id + } else { + const thread = await openai.beta.threads.retrieve(chatmessage.sessionId) + threadId = thread.id + } + + // Add message to thread + await openai.beta.threads.messages.create(threadId, { + role: 'user', + content: input + }) + + // Run assistant thread + const runThread = await openai.beta.threads.runs.create(threadId, { + assistant_id: retrievedAssistant.id + }) + + const usedTools: IUsedTool[] = [] + + const promise = (threadId: string, runId: string) => { + return new Promise((resolve, reject) => { + const timeout = setInterval(async () => { + const run = await openai.beta.threads.runs.retrieve(threadId, runId) + const state = run.status + if (state === 'completed') { + clearInterval(timeout) + resolve(state) + } else if (state === 'requires_action') { + if (run.required_action?.submit_tool_outputs.tool_calls) { + clearInterval(timeout) + const actions: ICommonObject[] = [] + run.required_action.submit_tool_outputs.tool_calls.forEach((item) => { + const functionCall = item.function + const args = JSON.parse(functionCall.arguments) + actions.push({ + tool: functionCall.name, + toolInput: args, + toolCallId: item.id + }) + }) + + const submitToolOutputs = [] + for (let i = 0; i < actions.length; i += 1) { + const tool = tools.find((tool: any) => tool.name === actions[i].tool) + if (!tool) continue + const toolOutput = await tool.call(actions[i].toolInput) + submitToolOutputs.push({ + tool_call_id: actions[i].toolCallId, + output: toolOutput + }) + usedTools.push({ + tool: tool.name, + toolInput: actions[i].toolInput, + toolOutput + }) + } + + if (submitToolOutputs.length) { + await openai.beta.threads.runs.submitToolOutputs(threadId, runId, { + tool_outputs: submitToolOutputs + }) + resolve(state) + } else { + reject( + new Error( + `Error processing thread: ${state}, Thread ID: ${threadId}, Run ID: ${runId}. submit_tool_outputs.tool_calls are empty` + ) + ) + } + } + } else if (state === 'cancelled' || state === 'expired' || state === 'failed') { + clearInterval(timeout) + reject(new Error(`Error processing thread: ${state}, Thread ID: ${threadId}, Run ID: ${runId}`)) + } + }, 500) + }) + } + + // Polling run status + let state = await promise(threadId, runThread.id) + while (state === 'requires_action') { + state = await promise(threadId, runThread.id) + } + + // List messages + const messages = await openai.beta.threads.messages.list(threadId) + const messageData = messages.data ?? [] + const assistantMessages = messageData.filter((msg) => msg.role === 'assistant') + if (!assistantMessages.length) return '' + + let returnVal = '' + for (let i = 0; i < assistantMessages[0].content.length; i += 1) { + if (assistantMessages[0].content[i].type === 'text') { + const content = assistantMessages[0].content[i] as MessageContentText + returnVal += content.text.value + + //TODO: handle annotations + } else { + const content = assistantMessages[0].content[i] as MessageContentImageFile + const fileId = content.image_file.file_id + const fileObj = await openai.files.retrieve(fileId) + const filePath = path.join(getUserHome(), '.flowise', 'openai-assistant', `${fileObj.filename}.png`) + + await downloadFile(fileObj, filePath, openAIApiKey) + + const bitmap = fsDefault.readFileSync(filePath) + const base64String = Buffer.from(bitmap).toString('base64') + + const imgHTML = `${fileObj.filename}
` + returnVal += imgHTML + } + } + + return { + text: returnVal, + usedTools, + assistant: { assistantId: openAIAssistantId, threadId, runId: runThread.id, messages: messageData } + } + } catch (error) { + throw new Error(error) + } } } @@ -221,4 +301,15 @@ const downloadFile = async (fileObj: any, filePath: string, openAIApiKey: string } } +const formatToOpenAIAssistantTool = (tool: any): OpenAI.Beta.AssistantCreateParams.AssistantToolsFunction => { + return { + type: 'function', + function: { + name: tool.name, + description: tool.description, + parameters: zodToJsonSchema(tool.schema) + } + } +} + module.exports = { nodeClass: OpenAIAssistant_Agents } diff --git a/packages/components/package.json b/packages/components/package.json index de5f264d..8c7c6703 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -72,7 +72,8 @@ "srt-parser-2": "^1.2.3", "vm2": "^3.9.19", "weaviate-ts-client": "^1.1.0", - "ws": "^8.9.0" + "ws": "^8.9.0", + "zod-to-json-schema": "^3.21.4" }, "devDependencies": { "@types/gulp": "4.0.9", diff --git a/packages/components/src/Interface.ts b/packages/components/src/Interface.ts index 5008813b..15b98770 100644 --- a/packages/components/src/Interface.ts +++ b/packages/components/src/Interface.ts @@ -126,6 +126,12 @@ export interface IMessage { type: MessageType } +export interface IUsedTool { + tool: string + toolInput: object + toolOutput: string | object +} + /** * Classes */ diff --git a/packages/server/package.json b/packages/server/package.json index 12d8bfbb..ab48decd 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -54,8 +54,8 @@ "express": "^4.17.3", "express-basic-auth": "^1.2.1", "express-rate-limit": "^6.9.0", - "flowise-components": "*", - "flowise-ui": "*", + "flowise-components": "1.4.0-rc.1", + "flowise-ui": "1.4.0-rc.1", "moment-timezone": "^0.5.34", "multer": "^1.4.5-lts.1", "mysql": "^2.18.1", diff --git a/packages/server/src/Interface.ts b/packages/server/src/Interface.ts index 1a5a694f..8d0965f4 100644 --- a/packages/server/src/Interface.ts +++ b/packages/server/src/Interface.ts @@ -29,6 +29,7 @@ export interface IChatMessage { content: string chatflowid: string sourceDocuments?: string + usedTools?: string chatType: string chatId: string memoryType?: string diff --git a/packages/server/src/commands/start.ts b/packages/server/src/commands/start.ts index 6bf7d699..cd874264 100644 --- a/packages/server/src/commands/start.ts +++ b/packages/server/src/commands/start.ts @@ -69,6 +69,10 @@ export default class Start extends Command { logger.error('uncaughtException: ', err) }) + process.on('unhandledRejection', (err) => { + logger.error('unhandledRejection: ', err) + }) + const { flags } = await this.parse(Start) if (flags.PORT) process.env.PORT = flags.PORT diff --git a/packages/server/src/database/entities/ChatMessage.ts b/packages/server/src/database/entities/ChatMessage.ts index 3efd7fbb..b51aa434 100644 --- a/packages/server/src/database/entities/ChatMessage.ts +++ b/packages/server/src/database/entities/ChatMessage.ts @@ -20,6 +20,9 @@ export class ChatMessage implements IChatMessage { @Column({ nullable: true, type: 'text' }) sourceDocuments?: string + @Column({ nullable: true, type: 'text' }) + usedTools?: string + @Column() chatType: string diff --git a/packages/server/src/database/migrations/mysql/1699481607341-AddUsedToolsToChatMessage.ts b/packages/server/src/database/migrations/mysql/1699481607341-AddUsedToolsToChatMessage.ts new file mode 100644 index 00000000..3ca170dc --- /dev/null +++ b/packages/server/src/database/migrations/mysql/1699481607341-AddUsedToolsToChatMessage.ts @@ -0,0 +1,12 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class AddUsedToolsToChatMessage1699481607341 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + const columnExists = await queryRunner.hasColumn('chat_message', 'usedTools') + if (!columnExists) queryRunner.query(`ALTER TABLE \`chat_message\` ADD COLUMN \`usedTools\` TEXT;`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE \`chat_message\` DROP COLUMN \`usedTools\`;`) + } +} diff --git a/packages/server/src/database/migrations/mysql/index.ts b/packages/server/src/database/migrations/mysql/index.ts index e372698c..4b7b8a95 100644 --- a/packages/server/src/database/migrations/mysql/index.ts +++ b/packages/server/src/database/migrations/mysql/index.ts @@ -7,6 +7,7 @@ import { AddApiConfig1694099200729 } from './1694099200729-AddApiConfig' import { AddAnalytic1694432361423 } from './1694432361423-AddAnalytic' import { AddChatHistory1694658767766 } from './1694658767766-AddChatHistory' import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEntity' +import { AddUsedToolsToChatMessage1699481607341 } from './1699481607341-AddUsedToolsToChatMessage' export const mysqlMigrations = [ Init1693840429259, @@ -17,5 +18,6 @@ export const mysqlMigrations = [ AddApiConfig1694099200729, AddAnalytic1694432361423, AddChatHistory1694658767766, - AddAssistantEntity1699325775451 + AddAssistantEntity1699325775451, + AddUsedToolsToChatMessage1699481607341 ] diff --git a/packages/server/src/database/migrations/postgres/1699481607341-AddUsedToolsToChatMessage.ts b/packages/server/src/database/migrations/postgres/1699481607341-AddUsedToolsToChatMessage.ts new file mode 100644 index 00000000..f9f893f8 --- /dev/null +++ b/packages/server/src/database/migrations/postgres/1699481607341-AddUsedToolsToChatMessage.ts @@ -0,0 +1,11 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class AddUsedToolsToChatMessage1699481607341 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "chat_message" ADD COLUMN IF NOT EXISTS "usedTools" TEXT;`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "chat_flow" DROP COLUMN "usedTools";`) + } +} diff --git a/packages/server/src/database/migrations/postgres/index.ts b/packages/server/src/database/migrations/postgres/index.ts index 75a1e523..75562c0b 100644 --- a/packages/server/src/database/migrations/postgres/index.ts +++ b/packages/server/src/database/migrations/postgres/index.ts @@ -7,6 +7,7 @@ import { AddApiConfig1694099183389 } from './1694099183389-AddApiConfig' import { AddAnalytic1694432361423 } from './1694432361423-AddAnalytic' import { AddChatHistory1694658756136 } from './1694658756136-AddChatHistory' import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEntity' +import { AddUsedToolsToChatMessage1699481607341 } from './1699481607341-AddUsedToolsToChatMessage' export const postgresMigrations = [ Init1693891895163, @@ -17,5 +18,6 @@ export const postgresMigrations = [ AddApiConfig1694099183389, AddAnalytic1694432361423, AddChatHistory1694658756136, - AddAssistantEntity1699325775451 + AddAssistantEntity1699325775451, + AddUsedToolsToChatMessage1699481607341 ] diff --git a/packages/server/src/database/migrations/sqlite/1699481607341-AddUsedToolsToChatMessage.ts b/packages/server/src/database/migrations/sqlite/1699481607341-AddUsedToolsToChatMessage.ts new file mode 100644 index 00000000..47334e56 --- /dev/null +++ b/packages/server/src/database/migrations/sqlite/1699481607341-AddUsedToolsToChatMessage.ts @@ -0,0 +1,20 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class AddUsedToolsToChatMessage1699481607341 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TABLE "temp_chat_message" ("id" varchar PRIMARY KEY NOT NULL, "role" varchar NOT NULL, "chatflowid" varchar NOT NULL, "content" text NOT NULL, "sourceDocuments" text, "usedTools" text, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "chatType" VARCHAR NOT NULL DEFAULT 'INTERNAL', "chatId" VARCHAR NOT NULL, "memoryType" VARCHAR, "sessionId" VARCHAR);` + ) + await queryRunner.query( + `INSERT INTO "temp_chat_message" ("id", "role", "chatflowid", "content", "sourceDocuments", "createdDate", "chatType", "chatId", "memoryType", "sessionId") SELECT "id", "role", "chatflowid", "content", "sourceDocuments", "createdDate", "chatType", "chatId", "memoryType", "sessionId" FROM "chat_message";` + ) + await queryRunner.query(`DROP TABLE "chat_message";`) + await queryRunner.query(`ALTER TABLE "temp_chat_message" RENAME TO "chat_message";`) + await queryRunner.query(`CREATE INDEX "IDX_e574527322272fd838f4f0f3d3" ON "chat_message" ("chatflowid") ;`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE IF EXISTS "temp_chat_message";`) + await queryRunner.query(`ALTER TABLE "chat_message" DROP COLUMN "usedTools";`) + } +} diff --git a/packages/server/src/database/migrations/sqlite/index.ts b/packages/server/src/database/migrations/sqlite/index.ts index ca4ecd57..4a14fc40 100644 --- a/packages/server/src/database/migrations/sqlite/index.ts +++ b/packages/server/src/database/migrations/sqlite/index.ts @@ -7,6 +7,7 @@ import { AddApiConfig1694090982460 } from './1694090982460-AddApiConfig' import { AddAnalytic1694432361423 } from './1694432361423-AddAnalytic' import { AddChatHistory1694657778173 } from './1694657778173-AddChatHistory' import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEntity' +import { AddUsedToolsToChatMessage1699481607341 } from './1699481607341-AddUsedToolsToChatMessage' export const sqliteMigrations = [ Init1693835579790, @@ -17,5 +18,6 @@ export const sqliteMigrations = [ AddApiConfig1694090982460, AddAnalytic1694432361423, AddChatHistory1694657778173, - AddAssistantEntity1699325775451 + AddAssistantEntity1699325775451, + AddUsedToolsToChatMessage1699481607341 ] diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index c9b25248..8968c3d4 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -49,7 +49,8 @@ import { replaceInputsWithConfig, getEncryptionKey, checkMemorySessionId, - clearSessionMemoryFromViewMessageDialog + clearSessionMemoryFromViewMessageDialog, + getUserHome } from './utils' import { cloneDeep, omit } from 'lodash' import { getDataSource } from './DataSource' @@ -670,6 +671,15 @@ export class App { const openai = new OpenAI({ apiKey: openAIApiKey }) const retrievedAssistant = await openai.beta.assistants.retrieve(req.params.id) + if (retrievedAssistant.file_ids && retrievedAssistant.file_ids.length) { + const files = [] + for (const file_id of retrievedAssistant.file_ids) { + const file = await openai.files.retrieve(file_id) + files.push(file) + } + ;(retrievedAssistant as any).files = files + } + return res.json(retrievedAssistant) }) @@ -701,46 +711,87 @@ export class App { const assistantDetails = JSON.parse(body.details) - if (!assistantDetails.id) { - try { - const credential = await this.AppDataSource.getRepository(Credential).findOneBy({ - id: body.credential - }) + try { + const credential = await this.AppDataSource.getRepository(Credential).findOneBy({ + id: body.credential + }) - if (!credential) return res.status(404).send(`Credential ${body.credential} not found`) + if (!credential) return res.status(404).send(`Credential ${body.credential} not found`) - // Decrpyt credentialData - const decryptedCredentialData = await decryptCredentialData(credential.encryptedData) - const openAIApiKey = decryptedCredentialData['openAIApiKey'] - if (!openAIApiKey) return res.status(404).send(`OpenAI ApiKey not found`) + // Decrpyt credentialData + const decryptedCredentialData = await decryptCredentialData(credential.encryptedData) + const openAIApiKey = decryptedCredentialData['openAIApiKey'] + if (!openAIApiKey) return res.status(404).send(`OpenAI ApiKey not found`) - const openai = new OpenAI({ apiKey: openAIApiKey }) + const openai = new OpenAI({ apiKey: openAIApiKey }) - let tools = [] - if (assistantDetails.tools) { - for (const tool of assistantDetails.tools ?? []) { - tools.push({ - type: tool - }) - } + let tools = [] + if (assistantDetails.tools) { + for (const tool of assistantDetails.tools ?? []) { + tools.push({ + type: tool + }) } + } + + if (assistantDetails.uploadFiles) { + // Base64 strings + let files: string[] = [] + const fileBase64 = assistantDetails.uploadFiles + if (fileBase64.startsWith('[') && fileBase64.endsWith(']')) { + files = JSON.parse(fileBase64) + } else { + files = [fileBase64] + } + + const uploadedFiles = [] + for (const file of files) { + const splitDataURI = file.split(',') + const filename = splitDataURI.pop()?.split(':')[1] ?? '' + const bf = Buffer.from(splitDataURI.pop() || '', 'base64') + const filePath = path.join(getUserHome(), '.flowise', 'openai-assistant', filename) + if (!fs.existsSync(filePath)) fs.writeFileSync(filePath, bf) + + const createdFile = await openai.files.create({ + file: fs.createReadStream(filePath), + purpose: 'assistants' + }) + uploadedFiles.push(createdFile) + + fs.unlinkSync(filePath) + } + assistantDetails.files = [...assistantDetails.files, ...uploadedFiles] + } + + if (!assistantDetails.id) { const newAssistant = await openai.beta.assistants.create({ name: assistantDetails.name, description: assistantDetails.description, instructions: assistantDetails.instructions, model: assistantDetails.model, - tools + tools, + file_ids: (assistantDetails.files ?? []).map((file: OpenAI.Files.FileObject) => file.id) + }) + assistantDetails.id = newAssistant.id + } else { + await openai.beta.assistants.update(assistantDetails.id, { + name: assistantDetails.name, + description: assistantDetails.description, + instructions: assistantDetails.instructions, + model: assistantDetails.model, + tools, + file_ids: (assistantDetails.files ?? []).map((file: OpenAI.Files.FileObject) => file.id) }) - - const newAssistantDetails = { - ...assistantDetails, - id: newAssistant.id - } - - body.details = JSON.stringify(newAssistantDetails) - } catch (error) { - return res.status(500).send(`Error creating new assistant: ${error}`) } + + const newAssistantDetails = { + ...assistantDetails + } + if (newAssistantDetails.uploadFiles) delete newAssistantDetails.uploadFiles + + body.details = JSON.stringify(newAssistantDetails) + } catch (error) { + return res.status(500).send(`Error creating new assistant: ${error}`) } const newAssistant = new Assistant() @@ -790,18 +841,50 @@ export class App { }) } } + + if (assistantDetails.uploadFiles) { + // Base64 strings + let files: string[] = [] + const fileBase64 = assistantDetails.uploadFiles + if (fileBase64.startsWith('[') && fileBase64.endsWith(']')) { + files = JSON.parse(fileBase64) + } else { + files = [fileBase64] + } + + const uploadedFiles = [] + for (const file of files) { + const splitDataURI = file.split(',') + const filename = splitDataURI.pop()?.split(':')[1] ?? '' + const bf = Buffer.from(splitDataURI.pop() || '', 'base64') + const filePath = path.join(getUserHome(), '.flowise', 'openai-assistant', filename) + if (!fs.existsSync(filePath)) fs.writeFileSync(filePath, bf) + + const createdFile = await openai.files.create({ + file: fs.createReadStream(filePath), + purpose: 'assistants' + }) + uploadedFiles.push(createdFile) + + fs.unlinkSync(filePath) + } + assistantDetails.files = [...assistantDetails.files, ...uploadedFiles] + } + await openai.beta.assistants.update(openAIAssistantId, { name: assistantDetails.name, description: assistantDetails.description, instructions: assistantDetails.instructions, model: assistantDetails.model, - tools + tools, + file_ids: (assistantDetails.files ?? []).map((file: OpenAI.Files.FileObject) => file.id) }) const newAssistantDetails = { ...assistantDetails, id: openAIAssistantId } + if (newAssistantDetails.uploadFiles) delete newAssistantDetails.uploadFiles const updateAssistant = new Assistant() body.details = JSON.stringify(newAssistantDetails) @@ -828,14 +911,13 @@ export class App { } try { - const body = req.body - const assistantDetails = JSON.parse(body.details) + const assistantDetails = JSON.parse(assistant.details) const credential = await this.AppDataSource.getRepository(Credential).findOneBy({ - id: body.credential + id: assistant.credential }) - if (!credential) return res.status(404).send(`Credential ${body.credential} not found`) + if (!credential) return res.status(404).send(`Credential ${assistant.credential} not found`) // Decrpyt credentialData const decryptedCredentialData = await decryptCredentialData(credential.encryptedData) @@ -844,11 +926,13 @@ export class App { const openai = new OpenAI({ apiKey: openAIApiKey }) + const results = await this.AppDataSource.getRepository(Assistant).delete({ id: req.params.id }) + await openai.beta.assistants.del(assistantDetails.id) - const results = await this.AppDataSource.getRepository(Assistant).delete({ id: req.params.id }) return res.json(results) - } catch (error) { + } catch (error: any) { + if (error.status === 404 && error.type === 'invalid_request_error') return res.send('OK') return res.status(500).send(`Error deleting assistant: ${error}`) } }) @@ -1389,6 +1473,7 @@ export class App { sessionId } if (result?.sourceDocuments) apiMessage.sourceDocuments = JSON.stringify(result.sourceDocuments) + if (result?.usedTools) apiMessage.usedTools = JSON.stringify(result.usedTools) await this.addChatMessage(apiMessage) logger.debug(`[server]: Finished running ${nodeToExecuteData.label} (${nodeToExecuteData.id})`) diff --git a/packages/ui/src/ui-component/dialog/SourceDocDialog.js b/packages/ui/src/ui-component/dialog/SourceDocDialog.js index 6bf8692f..f2a231e6 100644 --- a/packages/ui/src/ui-component/dialog/SourceDocDialog.js +++ b/packages/ui/src/ui-component/dialog/SourceDocDialog.js @@ -29,7 +29,7 @@ const SourceDocDialog = ({ show, dialogProps, onCancel }) => { aria-describedby='alert-dialog-description' > - Source Document + {dialogProps.title ?? 'Source Documents'} { time: chatmsg.createdDate } if (chatmsg.sourceDocuments) msg.sourceDocuments = JSON.parse(chatmsg.sourceDocuments) + if (chatmsg.usedTools) msg.usedTools = JSON.parse(chatmsg.usedTools) if (!Object.prototype.hasOwnProperty.call(obj, chatPK)) { obj[chatPK] = { @@ -251,6 +252,8 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => { type: chatmsg.role } if (chatmsg.sourceDocuments) obj.sourceDocuments = JSON.parse(chatmsg.sourceDocuments) + if (chatmsg.usedTools) obj.usedTools = JSON.parse(chatmsg.usedTools) + loadedMessages.push(obj) } setChatMessages(loadedMessages) @@ -315,8 +318,8 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => { window.open(data, '_blank') } - const onSourceDialogClick = (data) => { - setSourceDialogProps({ data }) + const onSourceDialogClick = (data, title) => { + setSourceDialogProps({ data, title }) setSourceDialogOpen(true) } @@ -599,6 +602,24 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => { width: '100%' }} > + {message.usedTools && ( +
+ {message.usedTools.map((tool, index) => { + return ( + onSourceDialogClick(tool, 'Used Tools')} + /> + ) + })} +
+ )}
{/* Messages are being rendered in Markdown format */} { const [assistantCredential, setAssistantCredential] = useState('') const [assistantInstructions, setAssistantInstructions] = useState('') const [assistantTools, setAssistantTools] = useState(['code_interpreter', 'retrieval']) + const [assistantFiles, setAssistantFiles] = useState([]) + const [uploadAssistantFiles, setUploadAssistantFiles] = useState('') + const [loading, setLoading] = useState(false) useEffect(() => { if (show) dispatch({ type: SHOW_CANVAS_DIALOG }) @@ -120,6 +117,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => { setAssistantModel(assistantDetails.model) setAssistantInstructions(assistantDetails.instructions) setAssistantTools(assistantDetails.tools ?? []) + setAssistantFiles(assistantDetails.files ?? []) } }, [getSpecificAssistantApi.data]) @@ -130,6 +128,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => { setAssistantDesc(getAssistantObjApi.data.description) setAssistantModel(getAssistantObjApi.data.model) setAssistantInstructions(getAssistantObjApi.data.instructions) + setAssistantFiles(getAssistantObjApi.data.files ?? []) let tools = [] if (getAssistantObjApi.data.tools && getAssistantObjApi.data.tools.length) { @@ -155,6 +154,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => { setAssistantModel(assistantDetails.model) setAssistantInstructions(assistantDetails.instructions) setAssistantTools(assistantDetails.tools ?? []) + setAssistantFiles(assistantDetails.files ?? []) } else if (dialogProps.type === 'EDIT' && dialogProps.assistantId) { // When assistant dialog is opened from OpenAIAssistant node in canvas getSpecificAssistantApi.request(dialogProps.assistantId) @@ -177,6 +177,8 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => { setAssistantModel('') setAssistantInstructions('') setAssistantTools(['code_interpreter', 'retrieval']) + setUploadAssistantFiles('') + setAssistantFiles([]) } return () => { @@ -190,11 +192,15 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => { setAssistantModel('') setAssistantInstructions('') setAssistantTools(['code_interpreter', 'retrieval']) + setUploadAssistantFiles('') + setAssistantFiles([]) + setLoading(false) } // eslint-disable-next-line react-hooks/exhaustive-deps }, [dialogProps]) const addNewAssistant = async () => { + setLoading(true) try { const assistantDetails = { id: openAIAssistantId, @@ -202,7 +208,9 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => { description: assistantDesc, model: assistantModel, instructions: assistantInstructions, - tools: assistantTools + tools: assistantTools, + files: assistantFiles, + uploadFiles: uploadAssistantFiles } const obj = { details: JSON.stringify(assistantDetails), @@ -226,6 +234,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => { }) onConfirm(createResp.data.id) } + setLoading(false) } catch (error) { const errorData = error.response.data || `${error.response.status}: ${error.response.statusText}` enqueueSnackbar({ @@ -241,18 +250,22 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => { ) } }) + setLoading(false) onCancel() } } const saveAssistant = async () => { + setLoading(true) try { const assistantDetails = { name: assistantName, description: assistantDesc, model: assistantModel, instructions: assistantInstructions, - tools: assistantTools + tools: assistantTools, + files: assistantFiles, + uploadFiles: uploadAssistantFiles } const obj = { details: JSON.stringify(assistantDetails), @@ -275,6 +288,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => { }) onConfirm(saveResp.data.id) } + setLoading(false) } catch (error) { const errorData = error.response.data || `${error.response.status}: ${error.response.statusText}` enqueueSnackbar({ @@ -290,6 +304,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => { ) } }) + setLoading(false) onCancel() } } @@ -341,6 +356,10 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => { } } + const onFileDeleteClick = async (fileId) => { + setAssistantFiles(assistantFiles.filter((file) => file.id !== fileId)) + } + const component = show ? ( { value={assistantTools ?? 'choose an option'} /> + + + + Knowledge Files + + + +
+ {assistantFiles.map((file, index) => ( +
+ {file.filename} + onFileDeleteClick(file.id)}> + + +
+ ))} +
+ setUploadAssistantFiles(newValue)} + value={uploadAssistantFiles ?? 'Choose a file to upload'} + /> + {dialogProps.type === 'EDIT' && ( @@ -529,6 +591,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => { + {loading && } ) : null diff --git a/packages/ui/src/views/canvas/NodeInputHandler.js b/packages/ui/src/views/canvas/NodeInputHandler.js index de865e31..7eb31bdb 100644 --- a/packages/ui/src/views/canvas/NodeInputHandler.js +++ b/packages/ui/src/views/canvas/NodeInputHandler.js @@ -22,6 +22,7 @@ import { isValidConnection } from 'utils/genericHelper' import { JsonEditorInput } from 'ui-component/json/JsonEditor' import { TooltipWithParser } from 'ui-component/tooltip/TooltipWithParser' import ToolDialog from 'views/tools/ToolDialog' +import AssistantDialog from 'views/assistants/AssistantDialog' import FormatPromptValuesDialog from 'ui-component/dialog/FormatPromptValuesDialog' import CredentialInputHandler from './CredentialInputHandler' @@ -31,7 +32,7 @@ import { getInputVariables } from 'utils/genericHelper' // const import { FLOWISE_CREDENTIAL_ID } from 'store/constant' -const EDITABLE_TOOLS = ['selectedTool'] +const EDITABLE_OPTIONS = ['selectedTool', 'selectedAssistant'] const CustomWidthTooltip = styled(({ className, ...props }) => )({ [`& .${tooltipClasses.tooltip}`]: { @@ -106,6 +107,14 @@ const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isA confirmButtonName: 'Save', toolId: inputValue }) + } else if (inputParamName === 'selectedAssistant') { + setAsyncOptionEditDialogProps({ + title: 'Edit Assistant', + type: 'EDIT', + cancelButtonName: 'Cancel', + confirmButtonName: 'Save', + assistantId: inputValue + }) } setAsyncOptionEditDialog(inputParamName) } @@ -118,6 +127,13 @@ const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isA cancelButtonName: 'Cancel', confirmButtonName: 'Add' }) + } else if (inputParamName === 'selectedAssistant') { + setAsyncOptionEditDialogProps({ + title: 'Add New Assistant', + type: 'ADD', + cancelButtonName: 'Cancel', + confirmButtonName: 'Add' + }) } setAsyncOptionEditDialog(inputParamName) } @@ -340,11 +356,11 @@ const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isA name={inputParam.name} nodeData={data} value={data.inputs[inputParam.name] ?? inputParam.default ?? 'choose an option'} - isCreateNewOption={EDITABLE_TOOLS.includes(inputParam.name)} + isCreateNewOption={EDITABLE_OPTIONS.includes(inputParam.name)} onSelect={(newValue) => (data.inputs[inputParam.name] = newValue)} onCreateNew={() => addAsyncOption(inputParam.name)} /> - {EDITABLE_TOOLS.includes(inputParam.name) && data.inputs[inputParam.name] && ( + {EDITABLE_OPTIONS.includes(inputParam.name) && data.inputs[inputParam.name] && ( )} setAsyncOptionEditDialog('')} onConfirm={onConfirmAsyncOption} > + setAsyncOptionEditDialog('')} + onConfirm={onConfirmAsyncOption} + >
) } diff --git a/packages/ui/src/views/chatmessage/ChatMessage.js b/packages/ui/src/views/chatmessage/ChatMessage.js index 3b9be768..0cf5695b 100644 --- a/packages/ui/src/views/chatmessage/ChatMessage.js +++ b/packages/ui/src/views/chatmessage/ChatMessage.js @@ -57,8 +57,8 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => { const getChatmessageApi = useApi(chatmessageApi.getInternalChatmessageFromChatflow) const getIsChatflowStreamingApi = useApi(chatflowsApi.getIsChatflowStreaming) - const onSourceDialogClick = (data) => { - setSourceDialogProps({ data }) + const onSourceDialogClick = (data, title) => { + setSourceDialogProps({ data, title }) setSourceDialogOpen(true) } @@ -139,7 +139,7 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => { setMessages((prevMessages) => [ ...prevMessages, - { message: text, sourceDocuments: data?.sourceDocuments, type: 'apiMessage' } + { message: text, sourceDocuments: data?.sourceDocuments, usedTools: data?.usedTools, type: 'apiMessage' } ]) } @@ -182,6 +182,7 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => { type: message.role } if (message.sourceDocuments) obj.sourceDocuments = JSON.parse(message.sourceDocuments) + if (message.usedTools) obj.usedTools = JSON.parse(message.usedTools) return obj }) setMessages((prevMessages) => [...prevMessages, ...loadedMessages]) @@ -284,6 +285,24 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => { Me )}
+ {message.usedTools && ( +
+ {message.usedTools.map((tool, index) => { + return ( + onSourceDialogClick(tool, 'Used Tools')} + /> + ) + })} +
+ )}
{/* Messages are being rendered in Markdown format */}