Merge branch 'main' into feature/allowed-domains

packages/components/credentials/GroqApi.credential.ts

@@ -0,0 +1,23 @@
+import { INodeParams, INodeCredential } from '../src/Interface'
+
+class GroqApi implements INodeCredential {
+    label: string
+    name: string
+    version: number
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'Groq API'
+        this.name = 'groqApi'
+        this.version = 1.0
+        this.inputs = [
+            {
+                label: 'Groq Api Key',
+                name: 'groqApiKey',
+                type: 'password'
+            }
+        ]
+    }
+}
+
+module.exports = { credClass: GroqApi }

packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts

@@ -1,17 +1,17 @@
 import { flatten } from 'lodash'
 import { AgentExecutor } from 'langchain/agents'
-import { pull } from 'langchain/hub'
+import { HumanMessage } from '@langchain/core/messages'
+import { ChatPromptTemplate, HumanMessagePromptTemplate } from '@langchain/core/prompts'
 import { Tool } from '@langchain/core/tools'
 import type { PromptTemplate } from '@langchain/core/prompts'
 import { BaseChatModel } from '@langchain/core/language_models/chat_models'
+import { pull } from 'langchain/hub'
 import { additionalCallbacks } from '../../../src/handler'
 import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { getBaseClasses } from '../../../src/utils'
 import { createReactAgent } from '../../../src/agents'
 import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
-import { HumanMessage } from '@langchain/core/messages'
 import { addImagesToMessages } from '../../../src/multiModalUtils'
-import { ChatPromptTemplate, HumanMessagePromptTemplate } from 'langchain/prompts'
 
 class MRKLAgentChat_Agents implements INode {
     label: string

packages/components/nodes/agents/MRKLAgentLLM/MRKLAgentLLM.ts

@@ -3,7 +3,7 @@ import { AgentExecutor } from 'langchain/agents'
 import { pull } from 'langchain/hub'
 import { Tool } from '@langchain/core/tools'
 import type { PromptTemplate } from '@langchain/core/prompts'
-import { BaseLanguageModel } from 'langchain/base_language'
+import { BaseLanguageModel } from '@langchain/core/language_models/base'
 import { additionalCallbacks } from '../../../src/handler'
 import { getBaseClasses } from '../../../src/utils'
 import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'

packages/components/nodes/agents/XMLAgent/XMLAgent.ts

@@ -0,0 +1,203 @@
+import { flatten } from 'lodash'
+import { ChainValues } from '@langchain/core/utils/types'
+import { AgentStep } from '@langchain/core/agents'
+import { RunnableSequence } from '@langchain/core/runnables'
+import { ChatOpenAI } from '@langchain/openai'
+import { Tool } from '@langchain/core/tools'
+import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder } from '@langchain/core/prompts'
+import { XMLAgentOutputParser } from 'langchain/agents/xml/output_parser'
+import { formatLogToMessage } from 'langchain/agents/format_scratchpad/log_to_message'
+import { getBaseClasses } from '../../../src/utils'
+import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
+import { AgentExecutor } from '../../../src/agents'
+//import { AgentExecutor } from "langchain/agents";
+
+const defaultSystemMessage = `You are a helpful assistant. Help the user answer any questions.
+
+You have access to the following tools:
+
+{tools}
+
+In order to use a tool, you can use <tool></tool> and <tool_input></tool_input> tags. You will then get back a response in the form <observation></observation>
+For example, if you have a tool called 'search' that could run a google search, in order to search for the weather in SF you would respond:
+
+<tool>search</tool><tool_input>weather in SF</tool_input>
+<observation>64 degrees</observation>
+
+When you are done, respond with a final answer between <final_answer></final_answer>. For example:
+
+<final_answer>The weather in SF is 64 degrees</final_answer>
+
+Begin!
+
+Previous Conversation:
+{chat_history}
+
+Question: {input}
+{agent_scratchpad}`
+
+class XMLAgent_Agents implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    sessionId?: string
+
+    constructor(fields?: { sessionId?: string }) {
+        this.label = 'XML Agent'
+        this.name = 'xmlAgent'
+        this.version = 1.0
+        this.type = 'XMLAgent'
+        this.category = 'Agents'
+        this.icon = 'xmlagent.svg'
+        this.description = `Agent that is designed for LLMs that are good for reasoning/writing XML (e.g: Anthropic Claude)`
+        this.baseClasses = [this.type, ...getBaseClasses(AgentExecutor)]
+        this.inputs = [
+            {
+                label: 'Tools',
+                name: 'tools',
+                type: 'Tool',
+                list: true
+            },
+            {
+                label: 'Memory',
+                name: 'memory',
+                type: 'BaseChatMemory'
+            },
+            {
+                label: 'Chat Model',
+                name: 'model',
+                type: 'BaseChatModel'
+            },
+            {
+                label: 'System Message',
+                name: 'systemMessage',
+                type: 'string',
+                warning: 'Prompt must include input variables: {tools}, {chat_history}, {input} and {agent_scratchpad}',
+                rows: 4,
+                default: defaultSystemMessage,
+                additionalParams: true
+            }
+        ]
+        this.sessionId = fields?.sessionId
+    }
+
+    async init(): Promise<any> {
+        return null
+    }
+
+    async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string | ICommonObject> {
+        const memory = nodeData.inputs?.memory as FlowiseMemory
+        const executor = await prepareAgent(nodeData, { sessionId: this.sessionId, chatId: options.chatId, input }, options.chatHistory)
+
+        const loggerHandler = new ConsoleCallbackHandler(options.logger)
+        const callbacks = await additionalCallbacks(nodeData, options)
+
+        let res: ChainValues = {}
+        let sourceDocuments: ICommonObject[] = []
+
+        if (options.socketIO && options.socketIOClientId) {
+            const handler = new CustomChainHandler(options.socketIO, options.socketIOClientId)
+            res = await executor.invoke({ input }, { callbacks: [loggerHandler, handler, ...callbacks] })
+            if (res.sourceDocuments) {
+                options.socketIO.to(options.socketIOClientId).emit('sourceDocuments', flatten(res.sourceDocuments))
+                sourceDocuments = res.sourceDocuments
+            }
+        } else {
+            res = await executor.invoke({ input }, { callbacks: [loggerHandler, ...callbacks] })
+            if (res.sourceDocuments) {
+                sourceDocuments = res.sourceDocuments
+            }
+        }
+
+        await memory.addChatMessages(
+            [
+                {
+                    text: input,
+                    type: 'userMessage'
+                },
+                {
+                    text: res?.output,
+                    type: 'apiMessage'
+                }
+            ],
+            this.sessionId
+        )
+
+        return sourceDocuments.length ? { text: res?.output, sourceDocuments: flatten(sourceDocuments) } : res?.output
+    }
+}
+
+const prepareAgent = async (
+    nodeData: INodeData,
+    flowObj: { sessionId?: string; chatId?: string; input?: string },
+    chatHistory: IMessage[] = []
+) => {
+    const model = nodeData.inputs?.model as ChatOpenAI
+    const memory = nodeData.inputs?.memory as FlowiseMemory
+    const systemMessage = nodeData.inputs?.systemMessage as string
+    let tools = nodeData.inputs?.tools
+    tools = flatten(tools)
+    const inputKey = memory.inputKey ? memory.inputKey : 'input'
+    const memoryKey = memory.memoryKey ? memory.memoryKey : 'chat_history'
+
+    let promptMessage = systemMessage ? systemMessage : defaultSystemMessage
+    if (memory.memoryKey) promptMessage = promptMessage.replaceAll('{chat_history}', `{${memory.memoryKey}}`)
+    if (memory.inputKey) promptMessage = promptMessage.replaceAll('{input}', `{${memory.inputKey}}`)
+
+    const prompt = ChatPromptTemplate.fromMessages([
+        HumanMessagePromptTemplate.fromTemplate(promptMessage),
+        new MessagesPlaceholder('agent_scratchpad')
+    ])
+
+    const missingVariables = ['tools', 'agent_scratchpad'].filter((v) => !prompt.inputVariables.includes(v))
+
+    if (missingVariables.length > 0) {
+        throw new Error(`Provided prompt is missing required input variables: ${JSON.stringify(missingVariables)}`)
+    }
+
+    const llmWithStop = model.bind({ stop: ['</tool_input>', '</final_answer>'] })
+
+    const messages = (await memory.getChatMessages(flowObj.sessionId, false, chatHistory)) as IMessage[]
+    let chatHistoryMsgTxt = ''
+    for (const message of messages) {
+        if (message.type === 'apiMessage') {
+            chatHistoryMsgTxt += `\\nAI:${message.message}`
+        } else if (message.type === 'userMessage') {
+            chatHistoryMsgTxt += `\\nHuman:${message.message}`
+        }
+    }
+
+    const runnableAgent = RunnableSequence.from([
+        {
+            [inputKey]: (i: { input: string; tools: Tool[]; steps: AgentStep[] }) => i.input,
+            agent_scratchpad: (i: { input: string; tools: Tool[]; steps: AgentStep[] }) => formatLogToMessage(i.steps),
+            tools: (_: { input: string; tools: Tool[]; steps: AgentStep[] }) =>
+                tools.map((tool: Tool) => `${tool.name}: ${tool.description}`),
+            [memoryKey]: (_: { input: string; tools: Tool[]; steps: AgentStep[] }) => chatHistoryMsgTxt
+        },
+        prompt,
+        llmWithStop,
+        new XMLAgentOutputParser()
+    ])
+
+    const executor = AgentExecutor.fromAgentAndTools({
+        agent: runnableAgent,
+        tools,
+        sessionId: flowObj?.sessionId,
+        chatId: flowObj?.chatId,
+        input: flowObj?.input,
+        isXML: true,
+        verbose: process.env.DEBUG === 'true' ? true : false
+    })
+
+    return executor
+}
+
+module.exports = { nodeClass: XMLAgent_Agents }

packages/components/nodes/agents/XMLAgent/xmlagent.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-file-type-xml" width="24" height="24" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round"><path stroke="none" d="M0 0h24v24H0z" fill="none"/><path d="M14 3v4a1 1 0 0 0 1 1h4" /><path d="M5 12v-7a2 2 0 0 1 2 -2h7l5 5v4" /><path d="M4 15l4 6" /><path d="M4 21l4 -6" /><path d="M19 15v6h3" /><path d="M11 21v-6l2.5 3l2.5 -3v6" /></svg>

packages/components/nodes/chains/LLMChain/LLMChain.ts

@@ -1,5 +1,6 @@
 import { BaseLanguageModel, BaseLanguageModelCallOptions } from '@langchain/core/language_models/base'
 import { BaseLLMOutputParser, BaseOutputParser } from '@langchain/core/output_parsers'
+import { HumanMessage } from '@langchain/core/messages'
 import { ChatPromptTemplate, FewShotPromptTemplate, PromptTemplate, HumanMessagePromptTemplate } from '@langchain/core/prompts'
 import { OutputFixingParser } from 'langchain/output_parsers'
 import { LLMChain } from 'langchain/chains'
@@ -10,7 +11,6 @@ import { checkInputs, Moderation, streamResponse } from '../../moderation/Modera
 import { formatResponse, injectOutputParser } from '../../outputparsers/OutputParserHelpers'
 import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
 import { addImagesToMessages } from '../../../src/multiModalUtils'
-import { HumanMessage } from 'langchain/schema'
 
 class LLMChain_Chains implements INode {
     label: string

packages/components/nodes/chatmodels/AWSBedrock/AWSChatBedrock.ts

@@ -95,6 +95,8 @@ class AWSChatBedrock_ChatModels implements INode {
                 name: 'model',
                 type: 'options',
                 options: [
+                    { label: 'anthropic.claude-3-sonnet', name: 'anthropic.claude-3-sonnet-20240229-v1:0' },
+                    { label: 'anthropic.claude-instant-v1', name: 'anthropic.claude-instant-v1' },
                     { label: 'anthropic.claude-instant-v1', name: 'anthropic.claude-instant-v1' },
                     { label: 'anthropic.claude-v1', name: 'anthropic.claude-v1' },
                     { label: 'anthropic.claude-v2', name: 'anthropic.claude-v2' },

packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts

@@ -43,6 +43,16 @@ class ChatAnthropic_ChatModels implements INode {
                 name: 'modelName',
                 type: 'options',
                 options: [
+                    {
+                        label: 'claude-3-opus',
+                        name: 'claude-3-opus-20240229',
+                        description: 'Most powerful model for highly complex tasks'
+                    },
+                    {
+                        label: 'claude-3-sonnet',
+                        name: 'claude-3-sonnet-20240229',
+                        description: 'Ideal balance of intelligence and speed for enterprise workloads'
+                    },
                     {
                         label: 'claude-2',
                         name: 'claude-2',

packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic_LlamaIndex.ts

@@ -37,6 +37,16 @@ class ChatAnthropic_LlamaIndex_ChatModels implements INode {
                 name: 'modelName',
                 type: 'options',
                 options: [
+                    {
+                        label: 'claude-3-opus',
+                        name: 'claude-3-opus-20240229',
+                        description: 'Most powerful model for highly complex tasks'
+                    },
+                    {
+                        label: 'claude-3-sonnet',
+                        name: 'claude-3-sonnet-20240229',
+                        description: 'Ideal balance of intelligence and speed for enterprise workloads'
+                    },
                     {
                         label: 'claude-2',
                         name: 'claude-2',

packages/components/nodes/chatmodels/ChatMistral/ChatMistral.ts

@@ -18,7 +18,7 @@ class ChatMistral_ChatModels implements INode {
     constructor() {
         this.label = 'ChatMistralAI'
         this.name = 'chatMistralAI'
-        this.version = 1.0
+        this.version = 2.0
         this.type = 'ChatMistralAI'
         this.icon = 'MistralAI.svg'
         this.category = 'Chat Models'
@@ -40,21 +40,9 @@ class ChatMistral_ChatModels implements INode {
             {
                 label: 'Model Name',
                 name: 'modelName',
-                type: 'options',
-                options: [
-                    {
-                        label: 'mistral-tiny',
-                        name: 'mistral-tiny'
-                    },
-                    {
-                        label: 'mistral-small',
-                        name: 'mistral-small'
-                    },
-                    {
-                        label: 'mistral-medium',
-                        name: 'mistral-medium'
-                    }
-                ],
+                type: 'string',
+                description:
+                    'Refer to <a target="_blank" href="https://docs.mistral.ai/guides/model-selection/">Model Selection</a> for more available models',
                 default: 'mistral-tiny'
             },
             {

packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts

@@ -7,9 +7,10 @@ import {
     ChatOpenAICallOptions
 } from '@langchain/openai'
 import { BaseChatModelParams } from '@langchain/core/language_models/chat_models'
-import { IMultiModalOption } from '../../../src'
-import { BaseMessageLike, LLMResult } from 'langchain/schema'
+import { BaseMessageLike } from '@langchain/core/messages'
 import { Callbacks } from '@langchain/core/callbacks/manager'
+import { LLMResult } from '@langchain/core/outputs'
+import { IMultiModalOption } from '../../../src'
 
 export class ChatOpenAI extends LangchainChatOpenAI {
     configuredModel: string

packages/components/nodes/chatmodels/Groq/Groq.ts

@@ -0,0 +1,80 @@
+import { BaseCache } from '@langchain/core/caches'
+import { ChatGroq, ChatGroqInput } from '@langchain/groq'
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+
+class Groq_ChatModels implements INode {
+    label: string
+    name: string
+    version: number
+    type: string
+    icon: string
+    category: string
+    description: string
+    baseClasses: string[]
+    credential: INodeParams
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'GroqChat'
+        this.name = 'groqChat'
+        this.version = 2.0
+        this.type = 'GroqChat'
+        this.icon = 'groq.png'
+        this.category = 'Chat Models'
+        this.description = 'Wrapper around Groq API with LPU Inference Engine'
+        this.baseClasses = [this.type, ...getBaseClasses(ChatGroq)]
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['groqApi'],
+            optional: true
+        }
+        this.inputs = [
+            {
+                label: 'Cache',
+                name: 'cache',
+                type: 'BaseCache',
+                optional: true
+            },
+            {
+                label: 'Model Name',
+                name: 'modelName',
+                type: 'string',
+                placeholder: 'ft:gpt-3.5-turbo:my-org:custom_suffix:id'
+            },
+            {
+                label: 'Temperature',
+                name: 'temperature',
+                type: 'number',
+                step: 0.1,
+                default: 0.9,
+                optional: true
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const modelName = nodeData.inputs?.modelName as string
+        const cache = nodeData.inputs?.cache as BaseCache
+        const temperature = nodeData.inputs?.temperature as string
+        const streaming = nodeData.inputs?.streaming as boolean
+
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const groqApiKey = getCredentialParam('groqApiKey', credentialData, nodeData)
+
+        const obj: ChatGroqInput = {
+            modelName,
+            temperature: parseFloat(temperature),
+            apiKey: groqApiKey,
+            streaming: streaming ?? true
+        }
+        if (cache) obj.cache = cache
+
+        const model = new ChatGroq(obj)
+        return model
+    }
+}
+
+module.exports = { nodeClass: Groq_ChatModels }

packages/components/nodes/memory/ConversationSummaryMemory/ConversationSummaryMemory.ts

@@ -1,8 +1,8 @@
 import { FlowiseSummaryMemory, IMessage, INode, INodeData, INodeParams, MemoryMethods } from '../../../src/Interface'
 import { convertBaseMessagetoIMessage, getBaseClasses } from '../../../src/utils'
-import { ConversationSummaryMemory, ConversationSummaryMemoryInput } from 'langchain/memory'
-import { BaseLanguageModel } from 'langchain/base_language'
+import { BaseLanguageModel } from '@langchain/core/language_models/base'
 import { BaseMessage } from '@langchain/core/messages'
+import { ConversationSummaryMemory, ConversationSummaryMemoryInput } from 'langchain/memory'
 
 class ConversationSummaryMemory_Memory implements INode {
     label: string

packages/components/nodes/utilities/CustomFunction/CustomFunction.ts

@@ -90,12 +90,17 @@ class CustomFunction_Utilities implements INode {
 
         // Some values might be a stringified JSON, parse it
         for (const key in inputVars) {
-            if (typeof inputVars[key] === 'string' && inputVars[key].startsWith('{') && inputVars[key].endsWith('}')) {
-                try {
-                    inputVars[key] = JSON.parse(inputVars[key])
-                } catch (e) {
-                    continue
+            let value = inputVars[key]
+            if (typeof value === 'string') {
+                value = handleEscapeCharacters(value, true)
+                if (value.startsWith('{') && value.endsWith('}')) {
+                    try {
+                        value = JSON.parse(value)
+                    } catch (e) {
+                        // ignore
+                    }
                 }
+                inputVars[key] = value
             }
         }
 
@@ -105,11 +110,7 @@ class CustomFunction_Utilities implements INode {
 
         if (Object.keys(inputVars).length) {
             for (const item in inputVars) {
-                let value = inputVars[item]
-                if (typeof value === 'string') {
-                    value = handleEscapeCharacters(value, true)
-                }
-                sandbox[`$${item}`] = value
+                sandbox[`$${item}`] = inputVars[item]
             }
         }
 

packages/components/nodes/utilities/IfElseFunction/IfElseFunction.ts

@@ -101,12 +101,17 @@ class IfElseFunction_Utilities implements INode {
 
         // Some values might be a stringified JSON, parse it
         for (const key in inputVars) {
-            if (typeof inputVars[key] === 'string' && inputVars[key].startsWith('{') && inputVars[key].endsWith('}')) {
-                try {
-                    inputVars[key] = JSON.parse(inputVars[key])
-                } catch (e) {
-                    continue
+            let value = inputVars[key]
+            if (typeof value === 'string') {
+                value = handleEscapeCharacters(value, true)
+                if (value.startsWith('{') && value.endsWith('}')) {
+                    try {
+                        value = JSON.parse(value)
+                    } catch (e) {
+                        // ignore
+                    }
                 }
+                inputVars[key] = value
             }
         }
 
@@ -116,11 +121,7 @@ class IfElseFunction_Utilities implements INode {
 
         if (Object.keys(inputVars).length) {
             for (const item in inputVars) {
-                let value = inputVars[item]
-                if (typeof value === 'string') {
-                    value = handleEscapeCharacters(value, true)
-                }
-                sandbox[`$${item}`] = value
+                sandbox[`$${item}`] = inputVars[item]
             }
         }
 

packages/components/package.json

@@ -33,6 +33,7 @@
         "@langchain/cohere": "^0.0.5",
         "@langchain/community": "^0.0.30",
         "@langchain/google-genai": "^0.0.10",
+        "@langchain/groq": "^0.0.2",
         "@langchain/mistralai": "^0.0.7",
         "@langchain/openai": "^0.0.14",
         "@langchain/pinecone": "^0.0.3",
@@ -72,7 +73,7 @@
         "lunary": "^0.6.16",
         "mammoth": "^1.5.1",
         "moment": "^2.29.3",
-        "mongodb": "^6.2.0",
+        "mongodb": "6.2.0",
         "mysql2": "^3.5.1",
         "node-fetch": "^2.6.11",
         "node-html-markdown": "^1.3.0",

packages/components/src/agents.ts

@@ -257,6 +257,8 @@ export class AgentExecutor extends BaseChain<ChainValues, AgentExecutorOutput> {
 
     input?: string
 
+    isXML?: boolean
+
     /**
      * How to handle errors raised by the agent's output parser.
         Defaults to `False`, which raises the error.
@@ -277,7 +279,7 @@ export class AgentExecutor extends BaseChain<ChainValues, AgentExecutorOutput> {
         return this.agent.returnValues
     }
 
-    constructor(input: AgentExecutorInput & { sessionId?: string; chatId?: string; input?: string }) {
+    constructor(input: AgentExecutorInput & { sessionId?: string; chatId?: string; input?: string; isXML?: boolean }) {
         let agent: BaseSingleActionAgent | BaseMultiActionAgent
         if (Runnable.isRunnable(input.agent)) {
             agent = new RunnableAgent({ runnable: input.agent })
@@ -305,13 +307,17 @@ export class AgentExecutor extends BaseChain<ChainValues, AgentExecutorOutput> {
         this.sessionId = input.sessionId
         this.chatId = input.chatId
         this.input = input.input
+        this.isXML = input.isXML
     }
 
-    static fromAgentAndTools(fields: AgentExecutorInput & { sessionId?: string; chatId?: string; input?: string }): AgentExecutor {
+    static fromAgentAndTools(
+        fields: AgentExecutorInput & { sessionId?: string; chatId?: string; input?: string; isXML?: boolean }
+    ): AgentExecutor {
         const newInstance = new AgentExecutor(fields)
         if (fields.sessionId) newInstance.sessionId = fields.sessionId
         if (fields.chatId) newInstance.chatId = fields.chatId
         if (fields.input) newInstance.input = fields.input
+        if (fields.isXML) newInstance.isXML = fields.isXML
         return newInstance
     }
 
@@ -405,12 +411,16 @@ export class AgentExecutor extends BaseChain<ChainValues, AgentExecutorOutput> {
                          * - flowConfig?: { sessionId?: string, chatId?: string, input?: string }
                          */
                         observation = tool
-                            ? // @ts-ignore
-                              await tool.call(action.toolInput, runManager?.getChild(), undefined, {
-                                  sessionId: this.sessionId,
-                                  chatId: this.chatId,
-                                  input: this.input
-                              })
+                            ? await (tool as any).call(
+                                  this.isXML && typeof action.toolInput === 'string' ? { input: action.toolInput } : action.toolInput,
+                                  runManager?.getChild(),
+                                  undefined,
+                                  {
+                                      sessionId: this.sessionId,
+                                      chatId: this.chatId,
+                                      input: this.input
+                                  }
+                              )
                             : `${action.tool} is not a valid tool, try another one.`
                     } catch (e) {
                         if (e instanceof ToolInputParsingException) {
@@ -526,12 +536,16 @@ export class AgentExecutor extends BaseChain<ChainValues, AgentExecutorOutput> {
                      * - tags?: string[]
                      * - flowConfig?: { sessionId?: string, chatId?: string, input?: string }
                      */
-                    // @ts-ignore
-                    observation = await tool.call(agentAction.toolInput, runManager?.getChild(), undefined, {
-                        sessionId: this.sessionId,
-                        chatId: this.chatId,
-                        input: this.input
-                    })
+                    observation = await (tool as any).call(
+                        this.isXML && typeof agentAction.toolInput === 'string' ? { input: agentAction.toolInput } : agentAction.toolInput,
+                        runManager?.getChild(),
+                        undefined,
+                        {
+                            sessionId: this.sessionId,
+                            chatId: this.chatId,
+                            input: this.input
+                        }
+                    )
                     if (observation?.includes(SOURCE_DOCUMENTS_PREFIX)) {
                         const observationArray = observation.split(SOURCE_DOCUMENTS_PREFIX)
                         observation = observationArray[0]

packages/components/src/multiModalUtils.ts

@@ -1,5 +1,5 @@
 import { ICommonObject, IFileUpload, IMultiModalOption, INodeData, MessageContentImageUrl } from './Interface'
-import { ChatOpenAI as LangchainChatOpenAI } from 'langchain/chat_models/openai'
+import { ChatOpenAI } from '../nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI'
 import path from 'path'
 import { getStoragePath } from './utils'
 import fs from 'fs'
@@ -12,7 +12,7 @@ export const addImagesToMessages = (
     const imageContent: MessageContentImageUrl[] = []
     let model = nodeData.inputs?.model
 
-    if (model instanceof LangchainChatOpenAI && multiModalOption) {
+    if (model instanceof ChatOpenAI && multiModalOption) {
         // Image Uploaded
         if (multiModalOption.image && multiModalOption.image.allowImageUploads && options?.uploads && options?.uploads.length > 0) {
             const imageUploads = getImageUploads(options.uploads)

packages/server/marketplaces/chatflows/Claude LLM.json

@@ -179,6 +179,16 @@
                         "name": "modelName",
                         "type": "options",
                         "options": [
+                            {
+                                "label": "claude-3-opus",
+                                "name": "claude-3-opus-20240229",
+                                "description": "Most powerful model for highly complex tasks"
+                            },
+                            {
+                                "label": "claude-3-sonnet",
+                                "name": "claude-3-sonnet-20240229",
+                                "description": "Ideal balance of intelligence and speed for enterprise workloads"
+                            },
                             {
                                 "label": "claude-2",
                                 "name": "claude-2",

packages/server/marketplaces/chatflows/OpenAI Assistant.json

@@ -5,12 +5,10 @@
     "badge": "NEW",
     "nodes": [
         {
-            "width": 300,
-            "height": 327,
             "id": "openAIAssistant_0",
             "position": {
-                "x": 895.3722263184736,
-                "y": 118.50795801755544
+                "x": 1237.914576178543,
+                "y": 140
             },
             "type": "customNode",
             "data": {
@@ -60,33 +58,36 @@
                 ],
                 "inputs": {
                     "selectedAssistant": "",
-                    "tools": ["{{calculator_0.data.instance}}", "{{serper_0.data.instance}}", "{{customTool_0.data.instance}}"]
+                    "tools": ["{{calculator_0.data.instance}}", "{{serper_0.data.instance}}", "{{customTool_0.data.instance}}"],
+                    "inputModeration": "",
+                    "disableFileDownload": ""
                 },
                 "outputAnchors": [
                     {
                         "id": "openAIAssistant_0-output-openAIAssistant-OpenAIAssistant",
                         "name": "openAIAssistant",
                         "label": "OpenAIAssistant",
+                        "description": "An agent that uses OpenAI Assistant API to pick the tool and args to call",
                         "type": "OpenAIAssistant"
                     }
                 ],
                 "outputs": {},
                 "selected": false
             },
+            "width": 300,
+            "height": 419,
             "selected": false,
             "dragging": false,
             "positionAbsolute": {
-                "x": 895.3722263184736,
-                "y": 118.50795801755544
+                "x": 1237.914576178543,
+                "y": 140
             }
         },
         {
-            "width": 300,
-            "height": 143,
             "id": "calculator_0",
             "position": {
-                "x": 454.74423492660145,
-                "y": -56.08375600705064
+                "x": 854.0341531341463,
+                "y": 48.134746169036475
             },
             "type": "customNode",
             "data": {
@@ -106,26 +107,75 @@
                         "id": "calculator_0-output-calculator-Calculator|Tool|StructuredTool|Runnable",
                         "name": "calculator",
                         "label": "Calculator",
+                        "description": "Perform calculations on response",
                         "type": "Calculator | Tool | StructuredTool | Runnable"
                     }
                 ],
                 "outputs": {},
                 "selected": false
             },
+            "width": 300,
+            "height": 142,
             "selected": false,
             "positionAbsolute": {
-                "x": 454.74423492660145,
-                "y": -56.08375600705064
+                "x": 854.0341531341463,
+                "y": 48.134746169036475
             },
             "dragging": false
         },
         {
+            "id": "serper_0",
+            "position": {
+                "x": 852.623106275503,
+                "y": 205.46647090775525
+            },
+            "type": "customNode",
+            "data": {
+                "id": "serper_0",
+                "label": "Serper",
+                "version": 1,
+                "name": "serper",
+                "type": "Serper",
+                "baseClasses": ["Serper", "Tool", "StructuredTool", "Runnable"],
+                "category": "Tools",
+                "description": "Wrapper around Serper.dev - Google Search API",
+                "inputParams": [
+                    {
+                        "label": "Connect Credential",
+                        "name": "credential",
+                        "type": "credential",
+                        "credentialNames": ["serperApi"],
+                        "id": "serper_0-input-credential-credential"
+                    }
+                ],
+                "inputAnchors": [],
+                "inputs": {},
+                "outputAnchors": [
+                    {
+                        "id": "serper_0-output-serper-Serper|Tool|StructuredTool|Runnable",
+                        "name": "serper",
+                        "label": "Serper",
+                        "description": "Wrapper around Serper.dev - Google Search API",
+                        "type": "Serper | Tool | StructuredTool | Runnable"
+                    }
+                ],
+                "outputs": {},
+                "selected": false
+            },
             "width": 300,
-            "height": 277,
+            "height": 276,
+            "selected": false,
+            "positionAbsolute": {
+                "x": 852.623106275503,
+                "y": 205.46647090775525
+            },
+            "dragging": false
+        },
+        {
             "id": "customTool_0",
             "position": {
-                "x": 454.43871855431365,
-                "y": 401.2171774551178
+                "x": 850.6759101766447,
+                "y": 496.68759375469654
             },
             "type": "customNode",
             "data": {
@@ -155,63 +205,19 @@
                         "id": "customTool_0-output-customTool-CustomTool|Tool|StructuredTool|Runnable",
                         "name": "customTool",
                         "label": "CustomTool",
+                        "description": "Use custom tool you've created in Flowise within chatflow",
                         "type": "CustomTool | Tool | StructuredTool | Runnable"
                     }
                 ],
                 "outputs": {},
                 "selected": false
             },
-            "selected": false,
-            "positionAbsolute": {
-                "x": 454.43871855431365,
-                "y": 401.2171774551178
-            },
-            "dragging": false
-        },
-        {
             "width": 300,
-            "height": 277,
-            "id": "serper_0",
-            "position": {
-                "x": 452.2514874331948,
-                "y": 99.6087116015905
-            },
-            "type": "customNode",
-            "data": {
-                "id": "serper_0",
-                "label": "Serper",
-                "version": 1,
-                "name": "serper",
-                "type": "Serper",
-                "baseClasses": ["Serper", "Tool", "StructuredTool", "Runnable"],
-                "category": "Tools",
-                "description": "Wrapper around Serper.dev - Google Search API",
-                "inputParams": [
-                    {
-                        "label": "Connect Credential",
-                        "name": "credential",
-                        "type": "credential",
-                        "credentialNames": ["serperApi"],
-                        "id": "serper_0-input-credential-credential"
-                    }
-                ],
-                "inputAnchors": [],
-                "inputs": {},
-                "outputAnchors": [
-                    {
-                        "id": "serper_0-output-serper-Serper|Tool|StructuredTool|Runnable",
-                        "name": "serper",
-                        "label": "Serper",
-                        "type": "Serper | Tool | StructuredTool | Runnable"
-                    }
-                ],
-                "outputs": {},
-                "selected": false
-            },
+            "height": 276,
             "selected": false,
             "positionAbsolute": {
-                "x": 452.2514874331948,
-                "y": 99.6087116015905
+                "x": 850.6759101766447,
+                "y": 496.68759375469654
             },
             "dragging": false
         }
@@ -223,10 +229,7 @@
             "target": "openAIAssistant_0",
             "targetHandle": "openAIAssistant_0-input-tools-Tool",
             "type": "buttonedge",
-            "id": "calculator_0-calculator_0-output-calculator-Calculator|Tool|StructuredTool|Runnable-openAIAssistant_0-openAIAssistant_0-input-tools-Tool",
-            "data": {
-                "label": ""
-            }
+            "id": "calculator_0-calculator_0-output-calculator-Calculator|Tool|StructuredTool|Runnable-openAIAssistant_0-openAIAssistant_0-input-tools-Tool"
         },
         {
             "source": "serper_0",
@@ -234,10 +237,7 @@
             "target": "openAIAssistant_0",
             "targetHandle": "openAIAssistant_0-input-tools-Tool",
             "type": "buttonedge",
-            "id": "serper_0-serper_0-output-serper-Serper|Tool|StructuredTool|Runnable-openAIAssistant_0-openAIAssistant_0-input-tools-Tool",
-            "data": {
-                "label": ""
-            }
+            "id": "serper_0-serper_0-output-serper-Serper|Tool|StructuredTool|Runnable-openAIAssistant_0-openAIAssistant_0-input-tools-Tool"
         },
         {
             "source": "customTool_0",
@@ -245,10 +245,7 @@
             "target": "openAIAssistant_0",
             "targetHandle": "openAIAssistant_0-input-tools-Tool",
             "type": "buttonedge",
-            "id": "customTool_0-customTool_0-output-customTool-CustomTool|Tool|StructuredTool|Runnable-openAIAssistant_0-openAIAssistant_0-input-tools-Tool",
-            "data": {
-                "label": ""
-            }
+            "id": "customTool_0-customTool_0-output-customTool-CustomTool|Tool|StructuredTool|Runnable-openAIAssistant_0-openAIAssistant_0-input-tools-Tool"
         }
     ]
 }

packages/server/marketplaces/chatflows/Query Engine.json

@@ -382,6 +382,16 @@
                         "name": "modelName",
                         "type": "options",
                         "options": [
+                            {
+                                "label": "claude-3-opus",
+                                "name": "claude-3-opus-20240229",
+                                "description": "Most powerful model for highly complex tasks"
+                            },
+                            {
+                                "label": "claude-3-sonnet",
+                                "name": "claude-3-sonnet-20240229",
+                                "description": "Ideal balance of intelligence and speed for enterprise workloads"
+                            },
                             {
                                 "label": "claude-2",
                                 "name": "claude-2",

packages/server/src/utils/index.ts

@@ -532,11 +532,45 @@ export const getVariableValue = (
                 variableDict[`{{${variableFullPath}}}`] = handleEscapeCharacters(convertChatHistoryToText(chatHistory), false)
             }
 
-            // Split by first occurrence of '.' to get just nodeId
-            const [variableNodeId, _] = variableFullPath.split('.')
+            // Resolve values with following case.
+            // 1: <variableNodeId>.data.instance
+            // 2: <variableNodeId>.data.instance.pathtokey
+            const variableFullPathParts = variableFullPath.split('.')
+            const variableNodeId = variableFullPathParts[0]
             const executedNode = reactFlowNodes.find((nd) => nd.id === variableNodeId)
             if (executedNode) {
-                const variableValue = get(executedNode.data, 'instance')
+                let variableValue = get(executedNode.data, 'instance')
+
+                // Handle path such as `<variableNodeId>.data.instance.key`
+                if (variableFullPathParts.length > 3) {
+                    let variableObj = null
+                    switch (typeof variableValue) {
+                        case 'string': {
+                            const unEscapedVariableValue = handleEscapeCharacters(variableValue, true)
+                            if (unEscapedVariableValue.startsWith('{') && unEscapedVariableValue.endsWith('}')) {
+                                try {
+                                    variableObj = JSON.parse(unEscapedVariableValue)
+                                } catch (e) {
+                                    // ignore
+                                }
+                            }
+                            break
+                        }
+                        case 'object': {
+                            variableObj = variableValue
+                            break
+                        }
+                        default:
+                            break
+                    }
+                    if (variableObj) {
+                        variableObj = get(variableObj, variableFullPathParts.slice(3))
+                        variableValue = handleEscapeCharacters(
+                            typeof variableObj === 'object' ? JSON.stringify(variableObj) : variableObj,
+                            false
+                        )
+                    }
+                }
                 if (isAcceptVariable) {
                     variableDict[`{{${variableFullPath}}}`] = variableValue
                 } else {
@@ -855,7 +889,8 @@ export const isFlowValidForStream = (reactFlowNodes: IReactFlowNode[], endingNod
             'chatAnthropic_LlamaIndex',
             'chatOllama',
             'awsChatBedrock',
-            'chatMistralAI'
+            'chatMistralAI',
+            'groqChat'
         ],
         LLMs: ['azureOpenAI', 'openAI', 'ollama']
     }

packages/ui/package.json

@@ -14,10 +14,10 @@
         "@emotion/cache": "^11.4.0",
         "@emotion/react": "^11.10.6",
         "@emotion/styled": "^11.10.6",
-        "@mui/icons-material": "^5.0.3",
-        "@mui/lab": "^5.0.0-alpha.156",
-        "@mui/material": "^5.15.0",
-        "@mui/x-data-grid": "^6.8.0",
+        "@mui/icons-material": "5.0.3",
+        "@mui/lab": "5.0.0-alpha.156",
+        "@mui/material": "5.15.0",
+        "@mui/x-data-grid": "6.8.0",
         "@tabler/icons": "^1.39.1",
         "@uiw/codemirror-theme-sublime": "^4.21.21",
         "@uiw/codemirror-theme-vscode": "^4.21.21",