farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-22 09:01:09 +03:00
Code Issues Packages Projects Releases Wiki Activity

add upload files and tool features

Browse Source
This commit is contained in:
Henry
2023-11-09 11:55:30 +00:00
parent 08eb8dd4e0
commit b607c1228f
19 changed files with 510 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/agents/OpenAIAssistant/OpenAIAssistant.ts
+169 -78
View File
@@ -1,4 +1,4 @@
import { ICommonObject, IDatabaseEntity, INode, INodeData, INodeOptionsValue, INodeParams } from '../../../src/Interface'
import { ICommonObject, IDatabaseEntity, INode, INodeData, INodeOptionsValue, INodeParams, IUsedTool } from '../../../src/Interface'
import OpenAI from 'openai'
import { DataSource } from 'typeorm'
import { getCredentialData, getCredentialParam, getUserHome } from '../../../src/utils'
@@ -6,6 +6,8 @@ import { MessageContentImageFile, MessageContentText } from 'openai/resources/be
import * as fsDefault from 'node:fs'
import * as path from 'node:path'
import fetch from 'node-fetch'
import { flatten } from 'lodash'
import { zodToJsonSchema } from 'zod-to-json-schema'
class OpenAIAssistant_Agents implements INode {
label: string
@@ -33,6 +35,12 @@ class OpenAIAssistant_Agents implements INode {
name: 'selectedAssistant',
type: 'asyncOptions',
loadMethod: 'listAssistants'
},
{
label: 'Allowed Tools',
name: 'tools',
type: 'Tool',
list: true
}
]
}
@@ -78,19 +86,28 @@ class OpenAIAssistant_Agents implements INode {
id: selectedAssistantId
})
if (!assistant) throw new Error(`Assistant ${selectedAssistantId} not found`)
if (!assistant) {
options.logger.error(`Assistant ${selectedAssistantId} not found`)
return
}
if (!sessionId && options.chatId) {
const chatmsg = await appDataSource.getRepository(databaseEntities['ChatMessage']).findOneBy({
chatId: options.chatId
})
if (!chatmsg) throw new Error(`Chat Message with Chat Id: ${options.chatId} not found`)
if (!chatmsg) {
options.logger.error(`Chat Message with Chat Id: ${options.chatId} not found`)
return
}
sessionId = chatmsg.sessionId
}
const credentialData = await getCredentialData(assistant.credential ?? '', options)
const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData)
if (!openAIApiKey) throw new Error(`OpenAI ApiKey not found`)
if (!openAIApiKey) {
options.logger.error(`OpenAI ApiKey not found`)
return
}
const openai = new OpenAI({ apiKey: openAIApiKey })
options.logger.info(`Clearing OpenAI Thread ${sessionId}`)
@@ -102,6 +119,9 @@ class OpenAIAssistant_Agents implements INode {
const selectedAssistantId = nodeData.inputs?.selectedAssistant as string
const appDataSource = options.appDataSource as DataSource
const databaseEntities = options.databaseEntities as IDatabaseEntity
let tools = nodeData.inputs?.tools
tools = flatten(tools)
const formattedTools = tools?.map((tool: any) => formatToOpenAIAssistantTool(tool)) ?? []
const assistant = await appDataSource.getRepository(databaseEntities['Assistant']).findOneBy({
id: selectedAssistantId
@@ -116,83 +136,143 @@ class OpenAIAssistant_Agents implements INode {
const openai = new OpenAI({ apiKey: openAIApiKey })
// Retrieve assistant
const assistantDetails = JSON.parse(assistant.details)
const openAIAssistantId = assistantDetails.id
const retrievedAssistant = await openai.beta.assistants.retrieve(openAIAssistantId)
try {
const assistantDetails = JSON.parse(assistant.details)
const openAIAssistantId = assistantDetails.id
const retrievedAssistant = await openai.beta.assistants.retrieve(openAIAssistantId)
const chatmessage = await appDataSource.getRepository(databaseEntities['ChatMessage']).findOneBy({
chatId: options.chatId
})
let threadId = ''
if (!chatmessage) {
const thread = await openai.beta.threads.create({})
threadId = thread.id
} else {
const thread = await openai.beta.threads.retrieve(chatmessage.sessionId)
threadId = thread.id
}
// Add message to thread
await openai.beta.threads.messages.create(threadId, {
role: 'user',
content: input
})
// Run assistant thread
const runThread = await openai.beta.threads.runs.create(threadId, {
assistant_id: retrievedAssistant.id
})
const promise = (threadId: string, runId: string) => {
return new Promise((resolve, reject) => {
const timeout = setInterval(async () => {
const run = await openai.beta.threads.runs.retrieve(threadId, runId)
const state = run.status
if (state === 'completed') {
clearInterval(timeout)
resolve(run)
} else if (state === 'cancelled' || state === 'expired' || state === 'failed') {
clearInterval(timeout)
reject(new Error(`Error processing thread: ${state}, Thread ID: ${threadId}, Run ID: ${runId}`))
}
}, 500)
})
}
// Polling run status
await promise(threadId, runThread.id)
// List messages
const messages = await openai.beta.threads.messages.list(threadId)
const messageData = messages.data ?? []
const assistantMessages = messageData.filter((msg) => msg.role === 'assistant')
if (!assistantMessages.length) return ''
let returnVal = ''
for (let i = 0; i < assistantMessages[0].content.length; i += 1) {
if (assistantMessages[0].content[i].type === 'text') {
const content = assistantMessages[0].content[i] as MessageContentText
returnVal += content.text.value
//TODO: handle annotations
} else {
const content = assistantMessages[0].content[i] as MessageContentImageFile
const fileId = content.image_file.file_id
const fileObj = await openai.files.retrieve(fileId)
const filePath = path.join(getUserHome(), '.flowise', 'openai-assistant', `${fileObj.filename}.png`)
await downloadFile(fileObj, filePath, openAIApiKey)
const bitmap = fsDefault.readFileSync(filePath)
const base64String = Buffer.from(bitmap).toString('base64')
const imgHTML = `<img src="data:image/png;base64,${base64String}" width="100%" height="max-content" alt="${fileObj.filename}" /><br/>`
returnVal += imgHTML
if (formattedTools.length) {
await openai.beta.assistants.update(openAIAssistantId, { tools: formattedTools })
}
}
return { text: returnVal, assistant: { assistantId: openAIAssistantId, threadId, runId: runThread.id, messages: messageData } }
const chatmessage = await appDataSource.getRepository(databaseEntities['ChatMessage']).findOneBy({
chatId: options.chatId
})
let threadId = ''
if (!chatmessage) {
const thread = await openai.beta.threads.create({})
threadId = thread.id
} else {
const thread = await openai.beta.threads.retrieve(chatmessage.sessionId)
threadId = thread.id
}
// Add message to thread
await openai.beta.threads.messages.create(threadId, {
role: 'user',
content: input
})
// Run assistant thread
const runThread = await openai.beta.threads.runs.create(threadId, {
assistant_id: retrievedAssistant.id
})
const usedTools: IUsedTool[] = []
const promise = (threadId: string, runId: string) => {
return new Promise((resolve, reject) => {
const timeout = setInterval(async () => {
const run = await openai.beta.threads.runs.retrieve(threadId, runId)
const state = run.status
if (state === 'completed') {
clearInterval(timeout)
resolve(state)
} else if (state === 'requires_action') {
if (run.required_action?.submit_tool_outputs.tool_calls) {
clearInterval(timeout)
const actions: ICommonObject[] = []
run.required_action.submit_tool_outputs.tool_calls.forEach((item) => {
const functionCall = item.function
const args = JSON.parse(functionCall.arguments)
actions.push({
tool: functionCall.name,
toolInput: args,
toolCallId: item.id
})
})
const submitToolOutputs = []
for (let i = 0; i < actions.length; i += 1) {
const tool = tools.find((tool: any) => tool.name === actions[i].tool)
if (!tool) continue
const toolOutput = await tool.call(actions[i].toolInput)
submitToolOutputs.push({
tool_call_id: actions[i].toolCallId,
output: toolOutput
})
usedTools.push({
tool: tool.name,
toolInput: actions[i].toolInput,
toolOutput
})
}
if (submitToolOutputs.length) {
await openai.beta.threads.runs.submitToolOutputs(threadId, runId, {
tool_outputs: submitToolOutputs
})
resolve(state)
} else {
reject(
new Error(
`Error processing thread: ${state}, Thread ID: ${threadId}, Run ID: ${runId}. submit_tool_outputs.tool_calls are empty`
)
)
}
}
} else if (state === 'cancelled' || state === 'expired' || state === 'failed') {
clearInterval(timeout)
reject(new Error(`Error processing thread: ${state}, Thread ID: ${threadId}, Run ID: ${runId}`))
}
}, 500)
})
}
// Polling run status
let state = await promise(threadId, runThread.id)
while (state === 'requires_action') {
state = await promise(threadId, runThread.id)
}
// List messages
const messages = await openai.beta.threads.messages.list(threadId)
const messageData = messages.data ?? []
const assistantMessages = messageData.filter((msg) => msg.role === 'assistant')
if (!assistantMessages.length) return ''
let returnVal = ''
for (let i = 0; i < assistantMessages[0].content.length; i += 1) {
if (assistantMessages[0].content[i].type === 'text') {
const content = assistantMessages[0].content[i] as MessageContentText
returnVal += content.text.value
//TODO: handle annotations
} else {
const content = assistantMessages[0].content[i] as MessageContentImageFile
const fileId = content.image_file.file_id
const fileObj = await openai.files.retrieve(fileId)
const filePath = path.join(getUserHome(), '.flowise', 'openai-assistant', `${fileObj.filename}.png`)
await downloadFile(fileObj, filePath, openAIApiKey)
const bitmap = fsDefault.readFileSync(filePath)
const base64String = Buffer.from(bitmap).toString('base64')
const imgHTML = `<img src="data:image/png;base64,${base64String}" width="100%" height="max-content" alt="${fileObj.filename}" /><br/>`
returnVal += imgHTML
}
}
return {
text: returnVal,
usedTools,
assistant: { assistantId: openAIAssistantId, threadId, runId: runThread.id, messages: messageData }
}
} catch (error) {
throw new Error(error)
}
}
}
@@ -221,4 +301,15 @@ const downloadFile = async (fileObj: any, filePath: string, openAIApiKey: string
}
}
const formatToOpenAIAssistantTool = (tool: any): OpenAI.Beta.AssistantCreateParams.AssistantToolsFunction => {
return {
type: 'function',
function: {
name: tool.name,
description: tool.description,
parameters: zodToJsonSchema(tool.schema)
}
}
}
module.exports = { nodeClass: OpenAIAssistant_Agents }
packages/components/package.json
+2 -1
View File
@@ -72,7 +72,8 @@
"srt-parser-2": "^1.2.3",
"vm2": "^3.9.19",
"weaviate-ts-client": "^1.1.0",
"ws": "^8.9.0"
"ws": "^8.9.0",
"zod-to-json-schema": "^3.21.4"
},
"devDependencies": {
"@types/gulp": "4.0.9",
packages/components/src/Interface.ts
+6
View File
@@ -126,6 +126,12 @@ export interface IMessage {
type: MessageType
}
export interface IUsedTool {
tool: string
toolInput: object
toolOutput: string | object
}
/**
* Classes
*/
packages/server/package.json
+2 -2
View File
@@ -54,8 +54,8 @@
"express": "^4.17.3",
"express-basic-auth": "^1.2.1",
"express-rate-limit": "^6.9.0",
"flowise-components": "*",
"flowise-ui": "*",
"flowise-components": "1.4.0-rc.1",
"flowise-ui": "1.4.0-rc.1",
"moment-timezone": "^0.5.34",
"multer": "^1.4.5-lts.1",
"mysql": "^2.18.1",
packages/server/src/Interface.ts
+1
View File
@@ -29,6 +29,7 @@ export interface IChatMessage {
content: string
chatflowid: string
sourceDocuments?: string
usedTools?: string
chatType: string
chatId: string
memoryType?: string
packages/server/src/commands/start.ts
+4
View File
@@ -69,6 +69,10 @@ export default class Start extends Command {
logger.error('uncaughtException: ', err)
})
process.on('unhandledRejection', (err) => {
logger.error('unhandledRejection: ', err)
})
const { flags } = await this.parse(Start)
if (flags.PORT) process.env.PORT = flags.PORT
packages/server/src/database/entities/ChatMessage.ts
+3
View File
@@ -20,6 +20,9 @@ export class ChatMessage implements IChatMessage {
@Column({ nullable: true, type: 'text' })
sourceDocuments?: string
@Column({ nullable: true, type: 'text' })
usedTools?: string
@Column()
chatType: string
packages/server/src/database/migrations/mysql/1699481607341-AddUsedToolsToChatMessage.ts
+12
View File
@@ -0,0 +1,12 @@
import { MigrationInterface, QueryRunner } from 'typeorm'
export class AddUsedToolsToChatMessage1699481607341 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
const columnExists = await queryRunner.hasColumn('chat_message', 'usedTools')
if (!columnExists) queryRunner.query(`ALTER TABLE \`chat_message\` ADD COLUMN \`usedTools\` TEXT;`)
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE \`chat_message\` DROP COLUMN \`usedTools\`;`)
}
}
packages/server/src/database/migrations/mysql/index.ts
+3 -1
View File
@@ -7,6 +7,7 @@ import { AddApiConfig1694099200729 } from './1694099200729-AddApiConfig'
import { AddAnalytic1694432361423 } from './1694432361423-AddAnalytic'
import { AddChatHistory1694658767766 } from './1694658767766-AddChatHistory'
import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEntity'
import { AddUsedToolsToChatMessage1699481607341 } from './1699481607341-AddUsedToolsToChatMessage'
export const mysqlMigrations = [
Init1693840429259,
@@ -17,5 +18,6 @@ export const mysqlMigrations = [
AddApiConfig1694099200729,
AddAnalytic1694432361423,
AddChatHistory1694658767766,
AddAssistantEntity1699325775451
AddAssistantEntity1699325775451,
AddUsedToolsToChatMessage1699481607341
]
packages/server/src/database/migrations/postgres/1699481607341-AddUsedToolsToChatMessage.ts
+11
View File
@@ -0,0 +1,11 @@
import { MigrationInterface, QueryRunner } from 'typeorm'
export class AddUsedToolsToChatMessage1699481607341 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "chat_message" ADD COLUMN IF NOT EXISTS "usedTools" TEXT;`)
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "chat_flow" DROP COLUMN "usedTools";`)
}
}
packages/server/src/database/migrations/postgres/index.ts
+3 -1
View File
@@ -7,6 +7,7 @@ import { AddApiConfig1694099183389 } from './1694099183389-AddApiConfig'
import { AddAnalytic1694432361423 } from './1694432361423-AddAnalytic'
import { AddChatHistory1694658756136 } from './1694658756136-AddChatHistory'
import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEntity'
import { AddUsedToolsToChatMessage1699481607341 } from './1699481607341-AddUsedToolsToChatMessage'
export const postgresMigrations = [
Init1693891895163,
@@ -17,5 +18,6 @@ export const postgresMigrations = [
AddApiConfig1694099183389,
AddAnalytic1694432361423,
AddChatHistory1694658756136,
AddAssistantEntity1699325775451
AddAssistantEntity1699325775451,
AddUsedToolsToChatMessage1699481607341
]
packages/server/src/database/migrations/sqlite/1699481607341-AddUsedToolsToChatMessage.ts
+20
View File
@@ -0,0 +1,20 @@
import { MigrationInterface, QueryRunner } from 'typeorm'
export class AddUsedToolsToChatMessage1699481607341 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`CREATE TABLE "temp_chat_message" ("id" varchar PRIMARY KEY NOT NULL, "role" varchar NOT NULL, "chatflowid" varchar NOT NULL, "content" text NOT NULL, "sourceDocuments" text, "usedTools" text, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "chatType" VARCHAR NOT NULL DEFAULT 'INTERNAL', "chatId" VARCHAR NOT NULL, "memoryType" VARCHAR, "sessionId" VARCHAR);`
)
await queryRunner.query(
`INSERT INTO "temp_chat_message" ("id", "role", "chatflowid", "content", "sourceDocuments", "createdDate", "chatType", "chatId", "memoryType", "sessionId") SELECT "id", "role", "chatflowid", "content", "sourceDocuments", "createdDate", "chatType", "chatId", "memoryType", "sessionId" FROM "chat_message";`
)
await queryRunner.query(`DROP TABLE "chat_message";`)
await queryRunner.query(`ALTER TABLE "temp_chat_message" RENAME TO "chat_message";`)
await queryRunner.query(`CREATE INDEX "IDX_e574527322272fd838f4f0f3d3" ON "chat_message" ("chatflowid") ;`)
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP TABLE IF EXISTS "temp_chat_message";`)
await queryRunner.query(`ALTER TABLE "chat_message" DROP COLUMN "usedTools";`)
}
}
packages/server/src/database/migrations/sqlite/index.ts
+3 -1
View File
@@ -7,6 +7,7 @@ import { AddApiConfig1694090982460 } from './1694090982460-AddApiConfig'
import { AddAnalytic1694432361423 } from './1694432361423-AddAnalytic'
import { AddChatHistory1694657778173 } from './1694657778173-AddChatHistory'
import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEntity'
import { AddUsedToolsToChatMessage1699481607341 } from './1699481607341-AddUsedToolsToChatMessage'
export const sqliteMigrations = [
Init1693835579790,
@@ -17,5 +18,6 @@ export const sqliteMigrations = [
AddApiConfig1694090982460,
AddAnalytic1694432361423,
AddChatHistory1694657778173,
AddAssistantEntity1699325775451
AddAssistantEntity1699325775451,
AddUsedToolsToChatMessage1699481607341
]
packages/server/src/index.ts
+121 -36
View File
@@ -49,7 +49,8 @@ import {
replaceInputsWithConfig,
getEncryptionKey,
checkMemorySessionId,
clearSessionMemoryFromViewMessageDialog
clearSessionMemoryFromViewMessageDialog,
getUserHome
} from './utils'
import { cloneDeep, omit } from 'lodash'
import { getDataSource } from './DataSource'
@@ -670,6 +671,15 @@ export class App {
const openai = new OpenAI({ apiKey: openAIApiKey })
const retrievedAssistant = await openai.beta.assistants.retrieve(req.params.id)
if (retrievedAssistant.file_ids && retrievedAssistant.file_ids.length) {
const files = []
for (const file_id of retrievedAssistant.file_ids) {
const file = await openai.files.retrieve(file_id)
files.push(file)
}
;(retrievedAssistant as any).files = files
}
return res.json(retrievedAssistant)
})
@@ -701,46 +711,87 @@ export class App {
const assistantDetails = JSON.parse(body.details)
if (!assistantDetails.id) {
try {
const credential = await this.AppDataSource.getRepository(Credential).findOneBy({
id: body.credential
})
try {
const credential = await this.AppDataSource.getRepository(Credential).findOneBy({
id: body.credential
})
if (!credential) return res.status(404).send(`Credential ${body.credential} not found`)
if (!credential) return res.status(404).send(`Credential ${body.credential} not found`)
// Decrpyt credentialData
const decryptedCredentialData = await decryptCredentialData(credential.encryptedData)
const openAIApiKey = decryptedCredentialData['openAIApiKey']
if (!openAIApiKey) return res.status(404).send(`OpenAI ApiKey not found`)
// Decrpyt credentialData
const decryptedCredentialData = await decryptCredentialData(credential.encryptedData)
const openAIApiKey = decryptedCredentialData['openAIApiKey']
if (!openAIApiKey) return res.status(404).send(`OpenAI ApiKey not found`)
const openai = new OpenAI({ apiKey: openAIApiKey })
const openai = new OpenAI({ apiKey: openAIApiKey })
let tools = []
if (assistantDetails.tools) {
for (const tool of assistantDetails.tools ?? []) {
tools.push({
type: tool
})
}
let tools = []
if (assistantDetails.tools) {
for (const tool of assistantDetails.tools ?? []) {
tools.push({
type: tool
})
}
}
if (assistantDetails.uploadFiles) {
// Base64 strings
let files: string[] = []
const fileBase64 = assistantDetails.uploadFiles
if (fileBase64.startsWith('[') && fileBase64.endsWith(']')) {
files = JSON.parse(fileBase64)
} else {
files = [fileBase64]
}
const uploadedFiles = []
for (const file of files) {
const splitDataURI = file.split(',')
const filename = splitDataURI.pop()?.split(':')[1] ?? ''
const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
const filePath = path.join(getUserHome(), '.flowise', 'openai-assistant', filename)
if (!fs.existsSync(filePath)) fs.writeFileSync(filePath, bf)
const createdFile = await openai.files.create({
file: fs.createReadStream(filePath),
purpose: 'assistants'
})
uploadedFiles.push(createdFile)
fs.unlinkSync(filePath)
}
assistantDetails.files = [...assistantDetails.files, ...uploadedFiles]
}
if (!assistantDetails.id) {
const newAssistant = await openai.beta.assistants.create({
name: assistantDetails.name,
description: assistantDetails.description,
instructions: assistantDetails.instructions,
model: assistantDetails.model,
tools
tools,
file_ids: (assistantDetails.files ?? []).map((file: OpenAI.Files.FileObject) => file.id)
})
assistantDetails.id = newAssistant.id
} else {
await openai.beta.assistants.update(assistantDetails.id, {
name: assistantDetails.name,
description: assistantDetails.description,
instructions: assistantDetails.instructions,
model: assistantDetails.model,
tools,
file_ids: (assistantDetails.files ?? []).map((file: OpenAI.Files.FileObject) => file.id)
})
const newAssistantDetails = {
...assistantDetails,
id: newAssistant.id
}
body.details = JSON.stringify(newAssistantDetails)
} catch (error) {
return res.status(500).send(`Error creating new assistant: ${error}`)
}
const newAssistantDetails = {
...assistantDetails
}
if (newAssistantDetails.uploadFiles) delete newAssistantDetails.uploadFiles
body.details = JSON.stringify(newAssistantDetails)
} catch (error) {
return res.status(500).send(`Error creating new assistant: ${error}`)
}
const newAssistant = new Assistant()
@@ -790,18 +841,50 @@ export class App {
})
}
}
if (assistantDetails.uploadFiles) {
// Base64 strings
let files: string[] = []
const fileBase64 = assistantDetails.uploadFiles
if (fileBase64.startsWith('[') && fileBase64.endsWith(']')) {
files = JSON.parse(fileBase64)
} else {
files = [fileBase64]
}
const uploadedFiles = []
for (const file of files) {
const splitDataURI = file.split(',')
const filename = splitDataURI.pop()?.split(':')[1] ?? ''
const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
const filePath = path.join(getUserHome(), '.flowise', 'openai-assistant', filename)
if (!fs.existsSync(filePath)) fs.writeFileSync(filePath, bf)
const createdFile = await openai.files.create({
file: fs.createReadStream(filePath),
purpose: 'assistants'
})
uploadedFiles.push(createdFile)
fs.unlinkSync(filePath)
}
assistantDetails.files = [...assistantDetails.files, ...uploadedFiles]
}
await openai.beta.assistants.update(openAIAssistantId, {
name: assistantDetails.name,
description: assistantDetails.description,
instructions: assistantDetails.instructions,
model: assistantDetails.model,
tools
tools,
file_ids: (assistantDetails.files ?? []).map((file: OpenAI.Files.FileObject) => file.id)
})
const newAssistantDetails = {
...assistantDetails,
id: openAIAssistantId
}
if (newAssistantDetails.uploadFiles) delete newAssistantDetails.uploadFiles
const updateAssistant = new Assistant()
body.details = JSON.stringify(newAssistantDetails)
@@ -828,14 +911,13 @@ export class App {
}
try {
const body = req.body
const assistantDetails = JSON.parse(body.details)
const assistantDetails = JSON.parse(assistant.details)
const credential = await this.AppDataSource.getRepository(Credential).findOneBy({
id: body.credential
id: assistant.credential
})
if (!credential) return res.status(404).send(`Credential ${body.credential} not found`)
if (!credential) return res.status(404).send(`Credential ${assistant.credential} not found`)
// Decrpyt credentialData
const decryptedCredentialData = await decryptCredentialData(credential.encryptedData)
@@ -844,11 +926,13 @@ export class App {
const openai = new OpenAI({ apiKey: openAIApiKey })
const results = await this.AppDataSource.getRepository(Assistant).delete({ id: req.params.id })
await openai.beta.assistants.del(assistantDetails.id)
const results = await this.AppDataSource.getRepository(Assistant).delete({ id: req.params.id })
return res.json(results)
} catch (error) {
} catch (error: any) {
if (error.status === 404 && error.type === 'invalid_request_error') return res.send('OK')
return res.status(500).send(`Error deleting assistant: ${error}`)
}
})
@@ -1389,6 +1473,7 @@ export class App {
sessionId
}
if (result?.sourceDocuments) apiMessage.sourceDocuments = JSON.stringify(result.sourceDocuments)
if (result?.usedTools) apiMessage.usedTools = JSON.stringify(result.usedTools)
await this.addChatMessage(apiMessage)
logger.debug(`[server]: Finished running ${nodeToExecuteData.label} (${nodeToExecuteData.id})`)
packages/ui/src/ui-component/dialog/SourceDocDialog.js
+1 -1
View File
@@ -29,7 +29,7 @@ const SourceDocDialog = ({ show, dialogProps, onCancel }) => {
aria-describedby='alert-dialog-description'
>
<DialogTitle sx={{ fontSize: '1rem' }} id='alert-dialog-title'>
Source Document
{dialogProps.title ?? 'Source Documents'}
</DialogTitle>
<DialogContent>
<ReactJson
packages/ui/src/ui-component/dialog/ViewMessagesDialog.js
+23 -2
View File
@@ -129,6 +129,7 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
time: chatmsg.createdDate
}
if (chatmsg.sourceDocuments) msg.sourceDocuments = JSON.parse(chatmsg.sourceDocuments)
if (chatmsg.usedTools) msg.usedTools = JSON.parse(chatmsg.usedTools)
if (!Object.prototype.hasOwnProperty.call(obj, chatPK)) {
obj[chatPK] = {
@@ -251,6 +252,8 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
type: chatmsg.role
}
if (chatmsg.sourceDocuments) obj.sourceDocuments = JSON.parse(chatmsg.sourceDocuments)
if (chatmsg.usedTools) obj.usedTools = JSON.parse(chatmsg.usedTools)
loadedMessages.push(obj)
}
setChatMessages(loadedMessages)
@@ -315,8 +318,8 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
window.open(data, '_blank')
}
const onSourceDialogClick = (data) => {
setSourceDialogProps({ data })
const onSourceDialogClick = (data, title) => {
setSourceDialogProps({ data, title })
setSourceDialogOpen(true)
}
@@ -599,6 +602,24 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
width: '100%'
}}
>
{message.usedTools && (
<div style={{ display: 'block', flexDirection: 'row', width: '100%' }}>
{message.usedTools.map((tool, index) => {
return (
<Chip
size='small'
key={index}
label={tool.tool}
component='a'
sx={{ mr: 1, mt: 1 }}
variant='outlined'
clickable
onClick={() => onSourceDialogClick(tool, 'Used Tools')}
/>
)
})}
</div>
)}
<div className='markdownanswer'>
{/* Messages are being rendered in Markdown format */}
<MemoizedReactMarkdown
packages/ui/src/views/assistants/AssistantDialog.js
+78 -15
View File
@@ -5,7 +5,7 @@ import { useDispatch } from 'react-redux'
import { enqueueSnackbar as enqueueSnackbarAction, closeSnackbar as closeSnackbarAction } from 'store/actions'
import { v4 as uuidv4 } from 'uuid'
import { Box, Typography, Button, Dialog, DialogActions, DialogContent, DialogTitle, Stack, OutlinedInput } from '@mui/material'
import { Box, Typography, Button, IconButton, Dialog, DialogActions, DialogContent, DialogTitle, Stack, OutlinedInput } from '@mui/material'
import { StyledButton } from 'ui-component/button/StyledButton'
import { TooltipWithParser } from 'ui-component/tooltip/TooltipWithParser'
@@ -13,6 +13,8 @@ import ConfirmDialog from 'ui-component/dialog/ConfirmDialog'
import { Dropdown } from 'ui-component/dropdown/Dropdown'
import { MultiDropdown } from 'ui-component/dropdown/MultiDropdown'
import CredentialInputHandler from 'views/canvas/CredentialInputHandler'
import { File } from 'ui-component/file/File'
import { BackdropLoader } from 'ui-component/loading/BackdropLoader'
// Icons
import { IconX } from '@tabler/icons'
@@ -29,29 +31,21 @@ import useNotifier from 'utils/useNotifier'
import { HIDE_CANVAS_DIALOG, SHOW_CANVAS_DIALOG } from 'store/actions'
const assistantAvailableModels = [
{
label: 'gpt-4',
name: 'gpt-4'
},
{
label: 'gpt-4-1106-preview',
name: 'gpt-4-1106-preview'
},
{
label: 'gpt-4-vision-preview',
name: 'gpt-4-vision-preview'
},
{
label: 'gpt-4-0613',
name: 'gpt-4-0613'
},
{
label: 'gpt-4-32k',
name: 'gpt-4-32k'
label: 'gpt-4-0314',
name: 'gpt-4-0314'
},
{
label: 'gpt-4-32k-0613',
name: 'gpt-4-32k-0613'
label: 'gpt-4',
name: 'gpt-4'
},
{
label: 'gpt-3.5-turbo',
@@ -100,6 +94,9 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
const [assistantCredential, setAssistantCredential] = useState('')
const [assistantInstructions, setAssistantInstructions] = useState('')
const [assistantTools, setAssistantTools] = useState(['code_interpreter', 'retrieval'])
const [assistantFiles, setAssistantFiles] = useState([])
const [uploadAssistantFiles, setUploadAssistantFiles] = useState('')
const [loading, setLoading] = useState(false)
useEffect(() => {
if (show) dispatch({ type: SHOW_CANVAS_DIALOG })
@@ -120,6 +117,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
setAssistantModel(assistantDetails.model)
setAssistantInstructions(assistantDetails.instructions)
setAssistantTools(assistantDetails.tools ?? [])
setAssistantFiles(assistantDetails.files ?? [])
}
}, [getSpecificAssistantApi.data])
@@ -130,6 +128,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
setAssistantDesc(getAssistantObjApi.data.description)
setAssistantModel(getAssistantObjApi.data.model)
setAssistantInstructions(getAssistantObjApi.data.instructions)
setAssistantFiles(getAssistantObjApi.data.files ?? [])
let tools = []
if (getAssistantObjApi.data.tools && getAssistantObjApi.data.tools.length) {
@@ -155,6 +154,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
setAssistantModel(assistantDetails.model)
setAssistantInstructions(assistantDetails.instructions)
setAssistantTools(assistantDetails.tools ?? [])
setAssistantFiles(assistantDetails.files ?? [])
} else if (dialogProps.type === 'EDIT' && dialogProps.assistantId) {
// When assistant dialog is opened from OpenAIAssistant node in canvas
getSpecificAssistantApi.request(dialogProps.assistantId)
@@ -177,6 +177,8 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
setAssistantModel('')
setAssistantInstructions('')
setAssistantTools(['code_interpreter', 'retrieval'])
setUploadAssistantFiles('')
setAssistantFiles([])
}
return () => {
@@ -190,11 +192,15 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
setAssistantModel('')
setAssistantInstructions('')
setAssistantTools(['code_interpreter', 'retrieval'])
setUploadAssistantFiles('')
setAssistantFiles([])
setLoading(false)
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [dialogProps])
const addNewAssistant = async () => {
setLoading(true)
try {
const assistantDetails = {
id: openAIAssistantId,
@@ -202,7 +208,9 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
description: assistantDesc,
model: assistantModel,
instructions: assistantInstructions,
tools: assistantTools
tools: assistantTools,
files: assistantFiles,
uploadFiles: uploadAssistantFiles
}
const obj = {
details: JSON.stringify(assistantDetails),
@@ -226,6 +234,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
})
onConfirm(createResp.data.id)
}
setLoading(false)
} catch (error) {
const errorData = error.response.data || `${error.response.status}: ${error.response.statusText}`
enqueueSnackbar({
@@ -241,18 +250,22 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
)
}
})
setLoading(false)
onCancel()
}
}
const saveAssistant = async () => {
setLoading(true)
try {
const assistantDetails = {
name: assistantName,
description: assistantDesc,
model: assistantModel,
instructions: assistantInstructions,
tools: assistantTools
tools: assistantTools,
files: assistantFiles,
uploadFiles: uploadAssistantFiles
}
const obj = {
details: JSON.stringify(assistantDetails),
@@ -275,6 +288,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
})
onConfirm(saveResp.data.id)
}
setLoading(false)
} catch (error) {
const errorData = error.response.data || `${error.response.status}: ${error.response.statusText}`
enqueueSnackbar({
@@ -290,6 +304,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
)
}
})
setLoading(false)
onCancel()
}
}
@@ -341,6 +356,10 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
}
}
const onFileDeleteClick = async (fileId) => {
setAssistantFiles(assistantFiles.filter((file) => file.id !== fileId))
}
const component = show ? (
<Dialog
fullWidth
@@ -513,6 +532,49 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
value={assistantTools ?? 'choose an option'}
/>
</Box>
<Box sx={{ p: 2 }}>
<Stack sx={{ position: 'relative' }} direction='row'>
<Typography variant='overline'>
Knowledge Files
<TooltipWithParser
style={{ marginLeft: 10 }}
title='Allow assistant to use the content from uploaded files for retrieval and code interpreter. MAX: 20 files'
/>
</Typography>
</Stack>
<div style={{ display: 'flex', flexDirection: 'row' }}>
{assistantFiles.map((file, index) => (
<div
key={index}
style={{
display: 'flex',
flexDirection: 'row',
alignItems: 'center',
width: 'max-content',
height: 'max-content',
borderRadius: 15,
background: 'rgb(254,252,191)',
paddingLeft: 15,
paddingRight: 15,
paddingTop: 5,
paddingBottom: 5,
marginRight: 10
}}
>
<span style={{ color: 'rgb(116,66,16)', marginRight: 10 }}>{file.filename}</span>
<IconButton sx={{ height: 15, width: 15, p: 0 }} onClick={() => onFileDeleteClick(file.id)}>
<IconX />
</IconButton>
</div>
))}
</div>
<File
key={uploadAssistantFiles}
fileType='*'
onChange={(newValue) => setUploadAssistantFiles(newValue)}
value={uploadAssistantFiles ?? 'Choose a file to upload'}
/>
</Box>
</DialogContent>
<DialogActions>
{dialogProps.type === 'EDIT' && (
@@ -529,6 +591,7 @@ const AssistantDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
</StyledButton>
</DialogActions>
<ConfirmDialog />
{loading && <BackdropLoader open={loading} />}
</Dialog>
) : null
packages/ui/src/views/canvas/NodeInputHandler.js
+26 -4
View File
@@ -22,6 +22,7 @@ import { isValidConnection } from 'utils/genericHelper'
import { JsonEditorInput } from 'ui-component/json/JsonEditor'
import { TooltipWithParser } from 'ui-component/tooltip/TooltipWithParser'
import ToolDialog from 'views/tools/ToolDialog'
import AssistantDialog from 'views/assistants/AssistantDialog'
import FormatPromptValuesDialog from 'ui-component/dialog/FormatPromptValuesDialog'
import CredentialInputHandler from './CredentialInputHandler'
@@ -31,7 +32,7 @@ import { getInputVariables } from 'utils/genericHelper'
// const
import { FLOWISE_CREDENTIAL_ID } from 'store/constant'
const EDITABLE_TOOLS = ['selectedTool']
const EDITABLE_OPTIONS = ['selectedTool', 'selectedAssistant']
const CustomWidthTooltip = styled(({ className, ...props }) => <Tooltip {...props} classes={{ popper: className }} />)({
[`& .${tooltipClasses.tooltip}`]: {
@@ -106,6 +107,14 @@ const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isA
confirmButtonName: 'Save',
toolId: inputValue
})
} else if (inputParamName === 'selectedAssistant') {
setAsyncOptionEditDialogProps({
title: 'Edit Assistant',
type: 'EDIT',
cancelButtonName: 'Cancel',
confirmButtonName: 'Save',
assistantId: inputValue
})
}
setAsyncOptionEditDialog(inputParamName)
}
@@ -118,6 +127,13 @@ const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isA
cancelButtonName: 'Cancel',
confirmButtonName: 'Add'
})
} else if (inputParamName === 'selectedAssistant') {
setAsyncOptionEditDialogProps({
title: 'Add New Assistant',
type: 'ADD',
cancelButtonName: 'Cancel',
confirmButtonName: 'Add'
})
}
setAsyncOptionEditDialog(inputParamName)
}
@@ -340,11 +356,11 @@ const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isA
name={inputParam.name}
nodeData={data}
value={data.inputs[inputParam.name] ?? inputParam.default ?? 'choose an option'}
isCreateNewOption={EDITABLE_TOOLS.includes(inputParam.name)}
isCreateNewOption={EDITABLE_OPTIONS.includes(inputParam.name)}
onSelect={(newValue) => (data.inputs[inputParam.name] = newValue)}
onCreateNew={() => addAsyncOption(inputParam.name)}
/>
{EDITABLE_TOOLS.includes(inputParam.name) && data.inputs[inputParam.name] && (
{EDITABLE_OPTIONS.includes(inputParam.name) && data.inputs[inputParam.name] && (
<IconButton
title='Edit'
color='primary'
@@ -361,11 +377,17 @@ const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isA
</>
)}
<ToolDialog
show={EDITABLE_TOOLS.includes(showAsyncOptionDialog)}
show={showAsyncOptionDialog === 'selectedTool'}
dialogProps={asyncOptionEditDialogProps}
onCancel={() => setAsyncOptionEditDialog('')}
onConfirm={onConfirmAsyncOption}
></ToolDialog>
<AssistantDialog
show={showAsyncOptionDialog === 'selectedAssistant'}
dialogProps={asyncOptionEditDialogProps}
onCancel={() => setAsyncOptionEditDialog('')}
onConfirm={onConfirmAsyncOption}
></AssistantDialog>
</div>
)
}
packages/ui/src/views/chatmessage/ChatMessage.js
+22 -3
View File
@@ -57,8 +57,8 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
const getChatmessageApi = useApi(chatmessageApi.getInternalChatmessageFromChatflow)
const getIsChatflowStreamingApi = useApi(chatflowsApi.getIsChatflowStreaming)
const onSourceDialogClick = (data) => {
setSourceDialogProps({ data })
const onSourceDialogClick = (data, title) => {
setSourceDialogProps({ data, title })
setSourceDialogOpen(true)
}
@@ -139,7 +139,7 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
setMessages((prevMessages) => [
...prevMessages,
{ message: text, sourceDocuments: data?.sourceDocuments, type: 'apiMessage' }
{ message: text, sourceDocuments: data?.sourceDocuments, usedTools: data?.usedTools, type: 'apiMessage' }
])
}
@@ -182,6 +182,7 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
type: message.role
}
if (message.sourceDocuments) obj.sourceDocuments = JSON.parse(message.sourceDocuments)
if (message.usedTools) obj.usedTools = JSON.parse(message.usedTools)
return obj
})
setMessages((prevMessages) => [...prevMessages, ...loadedMessages])
@@ -284,6 +285,24 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
<img src={userPNG} alt='Me' width='30' height='30' className='usericon' />
)}
<div style={{ display: 'flex', flexDirection: 'column', width: '100%' }}>
{message.usedTools && (
<div style={{ display: 'block', flexDirection: 'row', width: '100%' }}>
{message.usedTools.map((tool, index) => {
return (
<Chip
size='small'
key={index}
label={tool.tool}
component='a'
sx={{ mr: 1, mt: 1 }}
variant='outlined'
clickable
onClick={() => onSourceDialogClick(tool, 'Used Tools')}
/>
)
})}
</div>
)}
<div className='markdownanswer'>
{/* Messages are being rendered in Markdown format */}
<MemoizedReactMarkdown