mirror of
https://github.com/farcasclaudiu/Flowise.git
synced 2026-06-28 21:00:58 +03:00
Feature/lang graph (#2319)
* add langgraph * datasource: initial commit * datasource: datasource details and chunks * datasource: Document Store Node * more changes * Document Store - Base functionality * Document Store Loader Component * Document Store Loader Component * before merging the modularity PR * after merging the modularity PR * preview mode * initial draft PR * fixes * minor updates and fixes * preview with loader and splitter * preview with credential * show stored chunks * preview update... * edit config * save, preview and other changes * save, preview and other changes * save, process and other changes * save, process and other changes * alpha1 - for internal testing * rerouting urls * bug fix on new leader create * pagination support for chunks * delete document store * Update pnpm-lock.yaml * doc store card view * Update store files to use updated storage functions, Document Store Table View and other changes * ui changes * add expanded chunk dialog, improve ui * change throw Error to InternalError * Bug Fixes and removal of subFolder, adding of view chunks for store * lint fixes * merge changes * DocumentStoreStatus component * ui changes for doc store * add remove metadata key field, add custom document loader * add chatflows used doc store chips * add types/interfaces to DocumentStore Services * document loader list dialog title bar color change * update interfaces * Whereused Chatflow Name and Added chunkNo to retain order of created chunks. * use typeorm order chunkNo, ui changes * update tabler icons react * cleanup agents * add pysandbox tool * add abort functionality, loading next agent * add empty view svg * update chatflow tool with chatId * rename to agentflows * update worker for prompt input values * update dashboard to agentflows, agentcanvas * fix marketplace use template * add agentflow templates * resolve merge conflict * update baseURL --------- Co-authored-by: vinodkiran <vinodkiran@usa.net> Co-authored-by: Vinod Paidimarry <vinodkiran@outlook.in>
This commit is contained in:
Henry Heng
@@ -0,0 +1,454 @@
|
||||
import { flatten } from 'lodash'
|
||||
import { BaseChatModel } from '@langchain/core/language_models/chat_models'
|
||||
import { Runnable, RunnableConfig } from '@langchain/core/runnables'
|
||||
import { ChatPromptTemplate, MessagesPlaceholder, HumanMessagePromptTemplate } from '@langchain/core/prompts'
|
||||
import {
|
||||
ICommonObject,
|
||||
IMultiAgentNode,
|
||||
INode,
|
||||
INodeData,
|
||||
INodeParams,
|
||||
ITeamState,
|
||||
IVisionChatModal,
|
||||
MessageContentImageUrl
|
||||
} from '../../../src/Interface'
|
||||
import { Moderation } from '../../moderation/Moderation'
|
||||
import { z } from 'zod'
|
||||
import { StructuredTool } from '@langchain/core/tools'
|
||||
import { AgentExecutor, JsonOutputToolsParser, ToolCallingAgentOutputParser } from '../../../src/agents'
|
||||
import { ChatMistralAI } from '@langchain/mistralai'
|
||||
import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
|
||||
import { ChatAnthropic } from '../../chatmodels/ChatAnthropic/FlowiseChatAnthropic'
|
||||
import { ChatGoogleGenerativeAI } from '../../chatmodels/ChatGoogleGenerativeAI/FlowiseChatGoogleGenerativeAI'
|
||||
import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils'
|
||||
|
||||
const sysPrompt = `You are a supervisor tasked with managing a conversation between the following workers: {team_members}.
|
||||
Given the following user request, respond with the worker to act next.
|
||||
Each worker will perform a task and respond with their results and status.
|
||||
When finished, respond with FINISH.
|
||||
Select strategically to minimize the number of steps taken.`
|
||||
|
||||
const routerToolName = 'route'
|
||||
|
||||
class Supervisor_MultiAgents implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
credential: INodeParams
|
||||
inputs?: INodeParams[]
|
||||
badge?: string
|
||||
|
||||
constructor() {
|
||||
this.label = 'Supervisor'
|
||||
this.name = 'supervisor'
|
||||
this.version = 1.0
|
||||
this.type = 'Supervisor'
|
||||
this.icon = 'supervisor.svg'
|
||||
this.category = 'Multi Agents'
|
||||
this.baseClasses = [this.type]
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Supervisor Name',
|
||||
name: 'supervisorName',
|
||||
type: 'string',
|
||||
placeholder: 'Supervisor',
|
||||
default: 'Supervisor'
|
||||
},
|
||||
{
|
||||
label: 'Supervisor Prompt',
|
||||
name: 'supervisorPrompt',
|
||||
type: 'string',
|
||||
description: 'Prompt must contains {team_members}',
|
||||
rows: 4,
|
||||
default: sysPrompt,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Tool Calling Chat Model',
|
||||
name: 'model',
|
||||
type: 'BaseChatModel',
|
||||
description: `Only compatible with models that are capable of function calling: ChatOpenAI, ChatMistral, ChatAnthropic, ChatGoogleGenerativeAI, GroqChat. Best result with GPT-4 model`
|
||||
},
|
||||
{
|
||||
label: 'Recursion Limit',
|
||||
name: 'recursionLimit',
|
||||
type: 'number',
|
||||
description: 'Maximum number of times a call can recurse. If not provided, defaults to 100.',
|
||||
default: 100,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Input Moderation',
|
||||
description: 'Detect text that could generate harmful output and prevent it from being sent to the language model',
|
||||
name: 'inputModeration',
|
||||
type: 'Moderation',
|
||||
optional: true,
|
||||
list: true
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const llm = nodeData.inputs?.model as BaseChatModel
|
||||
const supervisorPrompt = nodeData.inputs?.supervisorPrompt as string
|
||||
const supervisorLabel = nodeData.inputs?.supervisorName as string
|
||||
const _recursionLimit = nodeData.inputs?.recursionLimit as string
|
||||
const recursionLimit = _recursionLimit ? parseFloat(_recursionLimit) : 100
|
||||
const moderations = (nodeData.inputs?.inputModeration as Moderation[]) ?? []
|
||||
|
||||
const abortControllerSignal = options.signal as AbortController
|
||||
|
||||
const workersNodes: IMultiAgentNode[] =
|
||||
nodeData.inputs?.workerNodes && nodeData.inputs?.workerNodes.length ? flatten(nodeData.inputs?.workerNodes) : []
|
||||
const workersNodeNames = workersNodes.map((node: IMultiAgentNode) => node.name)
|
||||
|
||||
if (!supervisorLabel) throw new Error('Supervisor name is required!')
|
||||
|
||||
const supervisorName = supervisorLabel.toLowerCase().replace(/\s/g, '_').trim()
|
||||
|
||||
let multiModalMessageContent: MessageContentImageUrl[] = []
|
||||
|
||||
async function createTeamSupervisor(llm: BaseChatModel, systemPrompt: string, members: string[]): Promise<Runnable> {
|
||||
const memberOptions = ['FINISH', ...members]
|
||||
|
||||
systemPrompt = systemPrompt.replaceAll('{team_members}', members.join(', '))
|
||||
|
||||
let userPrompt = `Given the conversation above, who should act next? Or should we FINISH? Select one of: ${memberOptions.join(
|
||||
', '
|
||||
)}`
|
||||
|
||||
const tool = new RouteTool({
|
||||
schema: z.object({
|
||||
reasoning: z.string(),
|
||||
next: z.enum(['FINISH', ...members]),
|
||||
instructions: z.string().describe('The specific instructions of the sub-task the next role should accomplish.')
|
||||
})
|
||||
})
|
||||
|
||||
let supervisor
|
||||
|
||||
if (llm instanceof ChatMistralAI) {
|
||||
let prompt = ChatPromptTemplate.fromMessages([
|
||||
['system', systemPrompt],
|
||||
new MessagesPlaceholder('messages'),
|
||||
['human', userPrompt]
|
||||
])
|
||||
|
||||
const messages = await processImageMessage(1, llm, prompt, nodeData, options)
|
||||
prompt = messages.prompt
|
||||
multiModalMessageContent = messages.multiModalMessageContent
|
||||
|
||||
// Force Mistral to use tool
|
||||
const modelWithTool = llm.bind({
|
||||
tools: [tool],
|
||||
tool_choice: 'any',
|
||||
signal: abortControllerSignal ? abortControllerSignal.signal : undefined
|
||||
})
|
||||
|
||||
const outputParser = new JsonOutputToolsParser()
|
||||
|
||||
supervisor = prompt
|
||||
.pipe(modelWithTool)
|
||||
.pipe(outputParser)
|
||||
.pipe((x) => {
|
||||
if (Array.isArray(x) && x.length) {
|
||||
const toolAgentAction = x[0]
|
||||
return {
|
||||
next: Object.keys(toolAgentAction.args).length ? toolAgentAction.args.next : 'FINISH',
|
||||
instructions: Object.keys(toolAgentAction.args).length
|
||||
? toolAgentAction.args.instructions
|
||||
: 'Conversation finished',
|
||||
team_members: members.join(', ')
|
||||
}
|
||||
} else {
|
||||
return {
|
||||
next: 'FINISH',
|
||||
instructions: 'Conversation finished',
|
||||
team_members: members.join(', ')
|
||||
}
|
||||
}
|
||||
})
|
||||
} else if (llm instanceof ChatAnthropic) {
|
||||
// Force Anthropic to use tool : https://docs.anthropic.com/claude/docs/tool-use#forcing-tool-use
|
||||
userPrompt = `Given the conversation above, who should act next? Or should we FINISH? Select one of: ${memberOptions.join(
|
||||
', '
|
||||
)}. Use the ${routerToolName} tool in your response.`
|
||||
|
||||
let prompt = ChatPromptTemplate.fromMessages([
|
||||
['system', systemPrompt],
|
||||
new MessagesPlaceholder('messages'),
|
||||
['human', userPrompt]
|
||||
])
|
||||
|
||||
const messages = await processImageMessage(1, llm, prompt, nodeData, options)
|
||||
prompt = messages.prompt
|
||||
multiModalMessageContent = messages.multiModalMessageContent
|
||||
|
||||
if (llm.bindTools === undefined) {
|
||||
throw new Error(`This agent only compatible with function calling models.`)
|
||||
}
|
||||
|
||||
const modelWithTool = llm.bindTools([tool])
|
||||
|
||||
const outputParser = new ToolCallingAgentOutputParser()
|
||||
|
||||
supervisor = prompt
|
||||
.pipe(modelWithTool)
|
||||
.pipe(outputParser)
|
||||
.pipe((x) => {
|
||||
if (Array.isArray(x) && x.length) {
|
||||
const toolAgentAction = x[0] as any
|
||||
return {
|
||||
next: toolAgentAction.toolInput.next,
|
||||
instructions: toolAgentAction.toolInput.instructions,
|
||||
team_members: members.join(', ')
|
||||
}
|
||||
} else if (typeof x === 'object' && 'returnValues' in x) {
|
||||
return {
|
||||
next: 'FINISH',
|
||||
instructions: x.returnValues?.output,
|
||||
team_members: members.join(', ')
|
||||
}
|
||||
} else {
|
||||
return {
|
||||
next: 'FINISH',
|
||||
instructions: 'Conversation finished',
|
||||
team_members: members.join(', ')
|
||||
}
|
||||
}
|
||||
})
|
||||
} else if (llm instanceof ChatOpenAI) {
|
||||
let prompt = ChatPromptTemplate.fromMessages([
|
||||
['system', systemPrompt],
|
||||
new MessagesPlaceholder('messages'),
|
||||
['human', userPrompt]
|
||||
])
|
||||
|
||||
const messages = await processImageMessage(1, llm, prompt, nodeData, options)
|
||||
prompt = messages.prompt
|
||||
multiModalMessageContent = messages.multiModalMessageContent
|
||||
|
||||
// Force OpenAI to use tool
|
||||
const modelWithTool = llm.bind({
|
||||
tools: [tool],
|
||||
tool_choice: { type: 'function', function: { name: routerToolName } },
|
||||
signal: abortControllerSignal ? abortControllerSignal.signal : undefined
|
||||
})
|
||||
|
||||
const outputParser = new ToolCallingAgentOutputParser()
|
||||
|
||||
supervisor = prompt
|
||||
.pipe(modelWithTool)
|
||||
.pipe(outputParser)
|
||||
.pipe((x) => {
|
||||
if (Array.isArray(x) && x.length) {
|
||||
const toolAgentAction = x[0] as any
|
||||
return {
|
||||
next: toolAgentAction.toolInput.next,
|
||||
instructions: toolAgentAction.toolInput.instructions,
|
||||
team_members: members.join(', ')
|
||||
}
|
||||
} else if (typeof x === 'object' && 'returnValues' in x) {
|
||||
return {
|
||||
next: 'FINISH',
|
||||
instructions: x.returnValues?.output,
|
||||
team_members: members.join(', ')
|
||||
}
|
||||
} else {
|
||||
return {
|
||||
next: 'FINISH',
|
||||
instructions: 'Conversation finished',
|
||||
team_members: members.join(', ')
|
||||
}
|
||||
}
|
||||
})
|
||||
} else if (llm instanceof ChatGoogleGenerativeAI) {
|
||||
/*
|
||||
* Gemini doesn't have system message and messages have to be alternate between model and user
|
||||
* So we have to place the system + human prompt at last
|
||||
*/
|
||||
let prompt = ChatPromptTemplate.fromMessages([
|
||||
['human', systemPrompt],
|
||||
['ai', ''],
|
||||
new MessagesPlaceholder('messages'),
|
||||
['ai', ''],
|
||||
['human', userPrompt]
|
||||
])
|
||||
|
||||
const messages = await processImageMessage(2, llm, prompt, nodeData, options)
|
||||
prompt = messages.prompt
|
||||
multiModalMessageContent = messages.multiModalMessageContent
|
||||
|
||||
if (llm.bindTools === undefined) {
|
||||
throw new Error(`This agent only compatible with function calling models.`)
|
||||
}
|
||||
const modelWithTool = llm.bindTools([tool])
|
||||
|
||||
const outputParser = new ToolCallingAgentOutputParser()
|
||||
|
||||
supervisor = prompt
|
||||
.pipe(modelWithTool)
|
||||
.pipe(outputParser)
|
||||
.pipe((x) => {
|
||||
if (Array.isArray(x) && x.length) {
|
||||
const toolAgentAction = x[0] as any
|
||||
return {
|
||||
next: toolAgentAction.toolInput.next,
|
||||
instructions: toolAgentAction.toolInput.instructions,
|
||||
team_members: members.join(', ')
|
||||
}
|
||||
} else if (typeof x === 'object' && 'returnValues' in x) {
|
||||
return {
|
||||
next: 'FINISH',
|
||||
instructions: x.returnValues?.output,
|
||||
team_members: members.join(', ')
|
||||
}
|
||||
} else {
|
||||
return {
|
||||
next: 'FINISH',
|
||||
instructions: 'Conversation finished',
|
||||
team_members: members.join(', ')
|
||||
}
|
||||
}
|
||||
})
|
||||
} else {
|
||||
let prompt = ChatPromptTemplate.fromMessages([
|
||||
['system', systemPrompt],
|
||||
new MessagesPlaceholder('messages'),
|
||||
['human', userPrompt]
|
||||
])
|
||||
|
||||
const messages = await processImageMessage(1, llm, prompt, nodeData, options)
|
||||
prompt = messages.prompt
|
||||
multiModalMessageContent = messages.multiModalMessageContent
|
||||
|
||||
if (llm.bindTools === undefined) {
|
||||
throw new Error(`This agent only compatible with function calling models.`)
|
||||
}
|
||||
const modelWithTool = llm.bindTools([tool])
|
||||
|
||||
const outputParser = new ToolCallingAgentOutputParser()
|
||||
|
||||
supervisor = prompt
|
||||
.pipe(modelWithTool)
|
||||
.pipe(outputParser)
|
||||
.pipe((x) => {
|
||||
if (Array.isArray(x) && x.length) {
|
||||
const toolAgentAction = x[0] as any
|
||||
return {
|
||||
next: toolAgentAction.toolInput.next,
|
||||
instructions: toolAgentAction.toolInput.instructions,
|
||||
team_members: members.join(', ')
|
||||
}
|
||||
} else if (typeof x === 'object' && 'returnValues' in x) {
|
||||
return {
|
||||
next: 'FINISH',
|
||||
instructions: x.returnValues?.output,
|
||||
team_members: members.join(', ')
|
||||
}
|
||||
} else {
|
||||
return {
|
||||
next: 'FINISH',
|
||||
instructions: 'Conversation finished',
|
||||
team_members: members.join(', ')
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
return supervisor
|
||||
}
|
||||
|
||||
const supervisorAgent = await createTeamSupervisor(llm, supervisorPrompt ? supervisorPrompt : sysPrompt, workersNodeNames)
|
||||
|
||||
const supervisorNode = async (state: ITeamState, config: RunnableConfig) =>
|
||||
await agentNode(
|
||||
{
|
||||
state,
|
||||
agent: supervisorAgent,
|
||||
abortControllerSignal
|
||||
},
|
||||
config
|
||||
)
|
||||
|
||||
const returnOutput: IMultiAgentNode = {
|
||||
node: supervisorNode,
|
||||
name: supervisorName ?? 'supervisor',
|
||||
label: supervisorLabel ?? 'Supervisor',
|
||||
type: 'supervisor',
|
||||
workers: workersNodeNames,
|
||||
recursionLimit,
|
||||
llm,
|
||||
moderations,
|
||||
multiModalMessageContent
|
||||
}
|
||||
|
||||
return returnOutput
|
||||
}
|
||||
}
|
||||
|
||||
async function agentNode(
|
||||
{ state, agent, abortControllerSignal }: { state: ITeamState; agent: AgentExecutor | Runnable; abortControllerSignal: AbortController },
|
||||
config: RunnableConfig
|
||||
) {
|
||||
try {
|
||||
if (abortControllerSignal.signal.aborted) {
|
||||
throw new Error('Aborted!')
|
||||
}
|
||||
const result = await agent.invoke({ ...state, signal: abortControllerSignal.signal }, config)
|
||||
return result
|
||||
} catch (error) {
|
||||
throw new Error('Aborted!')
|
||||
}
|
||||
}
|
||||
|
||||
const processImageMessage = async (
|
||||
index: number,
|
||||
llm: BaseChatModel,
|
||||
prompt: ChatPromptTemplate,
|
||||
nodeData: INodeData,
|
||||
options: ICommonObject
|
||||
) => {
|
||||
let multiModalMessageContent: MessageContentImageUrl[] = []
|
||||
|
||||
if (llmSupportsVision(llm)) {
|
||||
const visionChatModel = llm as IVisionChatModal
|
||||
multiModalMessageContent = await addImagesToMessages(nodeData, options, llm.multiModalOption)
|
||||
|
||||
if (multiModalMessageContent?.length) {
|
||||
visionChatModel.setVisionModel()
|
||||
|
||||
const msg = HumanMessagePromptTemplate.fromTemplate([...multiModalMessageContent])
|
||||
|
||||
prompt.promptMessages.splice(index, 0, msg)
|
||||
} else {
|
||||
visionChatModel.revertToOriginalModel()
|
||||
}
|
||||
}
|
||||
|
||||
return { prompt, multiModalMessageContent }
|
||||
}
|
||||
|
||||
class RouteTool extends StructuredTool {
|
||||
name = routerToolName
|
||||
|
||||
description = 'Select the worker to act next'
|
||||
|
||||
schema
|
||||
|
||||
constructor(fields: ICommonObject) {
|
||||
super()
|
||||
this.schema = fields.schema
|
||||
}
|
||||
|
||||
async _call(input: any) {
|
||||
return JSON.stringify(input)
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: Supervisor_MultiAgents }
|
||||
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-users-group" width="24" height="24" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round"><path stroke="none" d="M0 0h24v24H0z" fill="none"/><path d="M10 13a2 2 0 1 0 4 0a2 2 0 0 0 -4 0" /><path d="M8 21v-1a2 2 0 0 1 2 -2h4a2 2 0 0 1 2 2v1" /><path d="M15 5a2 2 0 1 0 4 0a2 2 0 0 0 -4 0" /><path d="M17 10h2a2 2 0 0 1 2 2v1" /><path d="M5 5a2 2 0 1 0 4 0a2 2 0 0 0 -4 0" /><path d="M3 13v-1a2 2 0 0 1 2 -2h2" /></svg>
|
||||
|
After Width: | Height: | Size: 559 B |
Reference in New Issue
Block a user