Feature/Code Interpreter (#3183)

* Base changes for ServerSide Events (instead of socket.io) * lint fixes * adding of interface and separate methods for streaming events * lint * first draft, handles both internal and external prediction end points. * lint fixes * additional internal end point for streaming and associated changes * return streamresponse as true to build agent flow * 1) JSON formatting for internal events 2) other fixes * 1) convert internal event to metadata to maintain consistency with external response * fix action and metadata streaming * fix for error when agent flow is aborted * prevent subflows from streaming and other code cleanup * prevent streaming from enclosed tools * add fix for preventing chaintool streaming * update lock file * add open when hidden to sse * Streaming errors * Streaming errors * add fix for showing error message * add code interpreter * add artifacts to view message dialog * Update pnpm-lock.yaml --------- Co-authored-by: Vinod Paidimarry <vinodkiran@outlook.in>
2026-06-28 23:01:09 +03:00 · 2024-09-17 08:44:56 +01:00
parent 26444ac3ae
commit b02f279e9d
21 changed files with 729 additions and 333 deletions
@@ -0,0 +1,269 @@
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { StructuredTool, ToolInputParsingException, ToolParams } from '@langchain/core/tools'
+import { CodeInterpreter } from '@e2b/code-interpreter'
+import { z } from 'zod'
+import { addSingleFileToStorage } from '../../../src/storageUtils'
+import { CallbackManager, CallbackManagerForToolRun, Callbacks, parseCallbackConfigArg } from '@langchain/core/callbacks/manager'
+import { RunnableConfig } from '@langchain/core/runnables'
+import { ARTIFACTS_PREFIX } from '../../../src/agents'
+
+const DESC = `Evaluates python code in a sandbox environment. \
+The environment is long running and exists across multiple executions. \
+You must send the whole script every time and print your outputs. \
+Script should be pure python code that can be evaluated. \
+It should be in python format NOT markdown. \
+The code should NOT be wrapped in backticks. \
+All python packages including requests, matplotlib, scipy, numpy, pandas, \
+etc are available. Create and display chart using "plt.show()".`
+const NAME = 'code_interpreter'
+
+class Code_Interpreter_Tools implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    badge: string
+    credential: INodeParams
+
+    constructor() {
+        this.label = 'Code Interpreter by E2B'
+        this.name = 'codeInterpreterE2B'
+        this.version = 1.0
+        this.type = 'CodeInterpreter'
+        this.icon = 'e2b.png'
+        this.category = 'Tools'
+        this.description = 'Execute code in a sandbox environment'
+        this.baseClasses = [this.type, 'Tool', ...getBaseClasses(E2BTool)]
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['E2BApi'],
+            optional: true
+        }
+        this.inputs = [
+            {
+                label: 'Tool Name',
+                name: 'toolName',
+                type: 'string',
+                description: 'Specify the name of the tool',
+                default: 'code_interpreter'
+            },
+            {
+                label: 'Tool Description',
+                name: 'toolDesc',
+                type: 'string',
+                rows: 4,
+                description: 'Specify the description of the tool',
+                default: DESC
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const toolDesc = nodeData.inputs?.toolDesc as string
+        const toolName = nodeData.inputs?.toolName as string
+
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const e2bApiKey = getCredentialParam('e2bApiKey', credentialData, nodeData)
+
+        return await E2BTool.initialize({
+            description: toolDesc ?? DESC,
+            name: toolName ?? NAME,
+            apiKey: e2bApiKey,
+            schema: z.object({
+                input: z.string().describe('Python code to be executed in the sandbox environment')
+            }),
+            chatflowid: options.chatflowid
+        })
+    }
+}
+
+type E2BToolParams = ToolParams
+type E2BToolInput = {
+    name: string
+    description: string
+    apiKey: string
+    schema: any
+    chatflowid: string
+    templateCodeInterpreterE2B?: string
+    domainCodeInterpreterE2B?: string
+}
+
+export class E2BTool extends StructuredTool {
+    static lc_name() {
+        return 'E2BTool'
+    }
+
+    name = NAME
+
+    description = DESC
+
+    instance: CodeInterpreter
+
+    apiKey: string
+
+    schema
+
+    chatflowid: string
+
+    flowObj: ICommonObject
+
+    templateCodeInterpreterE2B?: string
+    domainCodeInterpreterE2B?: string
+
+    constructor(options: E2BToolParams & E2BToolInput) {
+        super(options)
+        this.description = options.description
+        this.name = options.name
+        this.apiKey = options.apiKey
+        this.schema = options.schema
+        this.chatflowid = options.chatflowid
+        this.templateCodeInterpreterE2B = options.templateCodeInterpreterE2B
+        this.domainCodeInterpreterE2B = options.domainCodeInterpreterE2B
+    }
+
+    static async initialize(options: Partial<E2BToolParams> & E2BToolInput) {
+        return new this({
+            name: options.name,
+            description: options.description,
+            apiKey: options.apiKey,
+            schema: options.schema,
+            chatflowid: options.chatflowid,
+            templateCodeInterpreterE2B: options.templateCodeInterpreterE2B,
+            domainCodeInterpreterE2B: options.domainCodeInterpreterE2B
+        })
+    }
+
+    async call(
+        arg: z.infer<typeof this.schema>,
+        configArg?: RunnableConfig | Callbacks,
+        tags?: string[],
+        flowConfig?: { sessionId?: string; chatId?: string; input?: string; state?: ICommonObject }
+    ): Promise<string> {
+        const config = parseCallbackConfigArg(configArg)
+        if (config.runName === undefined) {
+            config.runName = this.name
+        }
+        let parsed
+        try {
+            parsed = await this.schema.parseAsync(arg)
+        } catch (e) {
+            throw new ToolInputParsingException(`Received tool input did not match expected schema`, JSON.stringify(arg))
+        }
+        const callbackManager_ = await CallbackManager.configure(
+            config.callbacks,
+            this.callbacks,
+            config.tags || tags,
+            this.tags,
+            config.metadata,
+            this.metadata,
+            { verbose: this.verbose }
+        )
+        const runManager = await callbackManager_?.handleToolStart(
+            this.toJSON(),
+            typeof parsed === 'string' ? parsed : JSON.stringify(parsed),
+            undefined,
+            undefined,
+            undefined,
+            undefined,
+            config.runName
+        )
+        let result
+        try {
+            result = await this._call(parsed, runManager, flowConfig)
+        } catch (e) {
+            await runManager?.handleToolError(e)
+            throw e
+        }
+        if (result && typeof result !== 'string') {
+            result = JSON.stringify(result)
+        }
+        await runManager?.handleToolEnd(result)
+        return result
+    }
+
+    // @ts-ignore
+    protected async _call(
+        arg: z.infer<typeof this.schema>,
+        _?: CallbackManagerForToolRun,
+        flowConfig?: { sessionId?: string; chatId?: string; input?: string }
+    ): Promise<string> {
+        flowConfig = { ...this.flowObj, ...flowConfig }
+        try {
+            if ('input' in arg) {
+                this.instance = await CodeInterpreter.create({ apiKey: this.apiKey })
+                const execution = await this.instance.notebook.execCell(arg?.input)
+
+                const artifacts = []
+                for (const result of execution.results) {
+                    for (const key in result) {
+                        if (!(result as any)[key]) continue
+
+                        if (key === 'png') {
+                            //@ts-ignore
+                            const pngData = Buffer.from(result.png, 'base64')
+
+                            const filename = `artifact_${Date.now()}.png`
+
+                            const res = await addSingleFileToStorage(
+                                'image/png',
+                                pngData,
+                                filename,
+                                this.chatflowid,
+                                flowConfig!.chatId as string
+                            )
+                            artifacts.push({ type: 'png', data: res })
+                        } else if (key === 'jpeg') {
+                            //@ts-ignore
+                            const jpegData = Buffer.from(result.jpeg, 'base64')
+
+                            const filename = `artifact_${Date.now()}.jpg`
+
+                            const res = await addSingleFileToStorage(
+                                'image/jpg',
+                                jpegData,
+                                filename,
+                                this.chatflowid,
+                                flowConfig!.chatId as string
+                            )
+                            artifacts.push({ type: 'jpeg', data: res })
+                        } else if (key === 'html' || key === 'markdown' || key === 'latex' || key === 'json' || key === 'javascript') {
+                            artifacts.push({ type: key, data: (result as any)[key] })
+                        } //TODO: support for pdf
+                    }
+                }
+
+                this.instance.close()
+
+                let output = ''
+
+                if (execution.text) output = execution.text
+                if (!execution.text && execution.logs.stdout.length) output = execution.logs.stdout.join('\n')
+
+                if (execution.error) {
+                    return `${execution.error.name}: ${execution.error.value}`
+                }
+
+                return artifacts.length > 0 ? output + ARTIFACTS_PREFIX + JSON.stringify(artifacts) : output
+            } else {
+                return 'No input provided'
+            }
+        } catch (e) {
+            if (this.instance) this.instance.close()
+            return typeof e === 'string' ? e : JSON.stringify(e, null, 2)
+        }
+    }
+
+    setFlowObject(flowObj: ICommonObject) {
+        this.flowObj = flowObj
+    }
+}
+
+module.exports = { nodeClass: Code_Interpreter_Tools }
@@ -1,151 +0,0 @@
-/* 
-* TODO: Implement codeInterpreter column to chat_message table
-import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
-import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
-import { StructuredTool, ToolParams } from '@langchain/core/tools'
-import { CodeInterpreter } from '@e2b/code-interpreter'
-import { z } from 'zod'
-
-const DESC = `Evaluates python code in a sandbox environment. \
-The environment is long running and exists across multiple executions. \
-You must send the whole script every time and print your outputs. \
-Script should be pure python code that can be evaluated. \
-It should be in python format NOT markdown. \
-The code should NOT be wrapped in backticks. \
-All python packages including requests, matplotlib, scipy, numpy, pandas, \
-etc are available. Create and display chart using "plt.show()".`
-const NAME = 'code_interpreter'
-
-class E2B_Tools implements INode {
-    label: string
-    name: string
-    version: number
-    description: string
-    type: string
-    icon: string
-    category: string
-    baseClasses: string[]
-    inputs: INodeParams[]
-    badge: string
-    credential: INodeParams
-
-    constructor() {
-        this.label = 'E2B'
-        this.name = 'e2b'
-        this.version = 1.0
-        this.type = 'E2B'
-        this.icon = 'e2b.png'
-        this.category = 'Tools'
-        this.description = 'Execute code in E2B Code Intepreter'
-        this.baseClasses = [this.type, 'Tool', ...getBaseClasses(E2BTool)]
-        this.credential = {
-            label: 'Connect Credential',
-            name: 'credential',
-            type: 'credential',
-            credentialNames: ['E2BApi']
-        }
-        this.inputs = [
-            {
-                label: 'Tool Name',
-                name: 'toolName',
-                type: 'string',
-                description: 'Specify the name of the tool',
-                default: 'code_interpreter'
-            },
-            {
-                label: 'Tool Description',
-                name: 'toolDesc',
-                type: 'string',
-                rows: 4,
-                description: 'Specify the description of the tool',
-                default: DESC
-            }
-        ]
-    }
-
-    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
-        const toolDesc = nodeData.inputs?.toolDesc as string
-        const toolName = nodeData.inputs?.toolName as string
-        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
-        const e2bApiKey = getCredentialParam('e2bApiKey', credentialData, nodeData)
-        const socketIO = options.socketIO
-        const socketIOClientId = options.socketIOClientId
-
-        return await E2BTool.initialize({
-            description: toolDesc ?? DESC,
-            name: toolName ?? NAME,
-            apiKey: e2bApiKey,
-            schema: z.object({
-                input: z.string().describe('Python code to be executed in the sandbox environment')
-            }),
-            socketIO,
-            socketIOClientId
-        })
-    }
-}
-
-type E2BToolParams = ToolParams & { instance: CodeInterpreter }
-
-export class E2BTool extends StructuredTool {
-    static lc_name() {
-        return 'E2BTool'
-    }
-
-    name = NAME
-
-    description = DESC
-
-    instance: CodeInterpreter
-
-    apiKey: string
-
-    schema
-
-    socketIO
-
-    socketIOClientId = ''
-
-    constructor(options: E2BToolParams & { name: string; description: string, apiKey: string, schema: any, socketIO: any, socketIOClientId: string}) {
-        super(options)
-        this.instance = options.instance
-        this.description = options.description
-        this.name = options.name
-        this.apiKey = options.apiKey
-        this.schema = options.schema
-        this.returnDirect = true
-        this.socketIO = options.socketIO
-        this.socketIOClientId = options.socketIOClientId
-    }
-
-    static async initialize(options: Partial<E2BToolParams> & { name: string; description: string, apiKey: string, schema: any, socketIO: any, socketIOClientId: string }) {
-        const instance = await CodeInterpreter.create({ apiKey: options.apiKey })
-        return new this({ instance, name: options.name, description: options.description, apiKey: options.apiKey, schema: options.schema, socketIO: options.socketIO, socketIOClientId: options.socketIOClientId})
-    }
-
-    async _call(args: any) {
-        try {
-            if ('input' in args) {
-                const execution = await this.instance.notebook.execCell(args?.input)
-                let imgHTML = ''
-                for (const result of execution.results) {
-                    if (result.png) {
-                        imgHTML += `\n\n<img src="data:image/png;base64,${result.png}" width="100%" height="max-content" alt="image" /><br/>`
-                    }
-                    if (result.jpeg) {
-                        imgHTML += `\n\n<img src="data:image/jpeg;base64,${result.jpeg}" width="100%" height="max-content" alt="image" /><br/>`
-                    }
-                }
-                const output = execution.text ? execution.text + imgHTML : imgHTML
-                if (this.socketIO && this.socketIOClientId) this.socketIO.to(this.socketIOClientId).emit('token', output)
-                return output
-            } else {
-                return 'No input provided'
-            }
-        } catch (e) {
-            return typeof e === 'string' ? e : JSON.stringify(e, null, 2)
-        }
-    }
-}
-
-module.exports = { nodeClass: E2B_Tools }
-*/
@@ -1,127 +0,0 @@
-import { INode, INodeData, INodeParams } from '../../../src/Interface'
-import { getBaseClasses } from '../../../src/utils'
-import { loadPyodide, type PyodideInterface } from 'pyodide'
-import { Tool, ToolParams } from '@langchain/core/tools'
-import * as path from 'path'
-import { getUserHome } from '../../../src/utils'
-
-let pyodideInstance: PyodideInterface | undefined
-const DESC = `Evaluates python code in a sandbox environment. The environment resets on every execution. You must send the whole script every time and print your outputs. Script should be pure python code that can be evaluated. Use only packages available in Pyodide.`
-const NAME = 'python_interpreter'
-
-async function LoadPyodide(): Promise<PyodideInterface> {
-    if (pyodideInstance === undefined) {
-        const obj = { packageCacheDir: path.join(getUserHome(), '.flowise', 'pyodideCacheDir') }
-        pyodideInstance = await loadPyodide(obj)
-    }
-    return pyodideInstance
-}
-
-class PythonInterpreter_Tools implements INode {
-    label: string
-    name: string
-    version: number
-    description: string
-    type: string
-    icon: string
-    category: string
-    baseClasses: string[]
-    inputs: INodeParams[]
-    badge: string
-
-    constructor() {
-        this.label = 'Python Interpreter'
-        this.name = 'pythonInterpreter'
-        this.version = 1.0
-        this.type = 'PythonInterpreter'
-        this.icon = 'python.svg'
-        this.category = 'Tools'
-        this.description = 'Execute python code in Pyodide sandbox environment'
-        this.baseClasses = [this.type, 'Tool', ...getBaseClasses(PythonInterpreterTool)]
-        this.inputs = [
-            {
-                label: 'Tool Name',
-                name: 'toolName',
-                type: 'string',
-                description: 'Specify the name of the tool',
-                default: 'python_interpreter'
-            },
-            {
-                label: 'Tool Description',
-                name: 'toolDesc',
-                type: 'string',
-                rows: 4,
-                description: 'Specify the description of the tool',
-                default: DESC
-            }
-        ]
-    }
-
-    async init(nodeData: INodeData): Promise<any> {
-        const toolDesc = nodeData.inputs?.toolDesc as string
-        const toolName = nodeData.inputs?.toolName as string
-
-        return await PythonInterpreterTool.initialize({
-            description: toolDesc ?? DESC,
-            name: toolName ?? NAME
-        })
-    }
-}
-
-type PythonInterpreterToolParams = Parameters<typeof loadPyodide>[0] &
-    ToolParams & {
-        instance: PyodideInterface
-    }
-
-export class PythonInterpreterTool extends Tool {
-    static lc_name() {
-        return 'PythonInterpreterTool'
-    }
-
-    name = NAME
-
-    description = DESC
-
-    pyodideInstance: PyodideInterface
-
-    stdout = ''
-
-    stderr = ''
-
-    constructor(options: PythonInterpreterToolParams & { name: string; description: string }) {
-        super(options)
-        this.description = options.description
-        this.name = options.name
-        this.pyodideInstance = options.instance
-        this.pyodideInstance.setStderr({
-            batched: (text: string) => {
-                this.stderr += text
-            }
-        })
-        this.pyodideInstance.setStdout({
-            batched: (text: string) => {
-                this.stdout += text
-            }
-        })
-    }
-
-    static async initialize(options: Partial<PythonInterpreterToolParams> & { name: string; description: string }) {
-        const instance = await LoadPyodide()
-        return new this({ instance, name: options.name, description: options.description })
-    }
-
-    async _call(script: string) {
-        this.stdout = ''
-        this.stderr = ''
-
-        try {
-            await this.pyodideInstance.loadPackagesFromImports(script)
-            await this.pyodideInstance.runPythonAsync(script)
-            return JSON.stringify({ stdout: this.stdout, stderr: this.stderr }, null, 2)
-        } catch (e) {
-            return typeof e === 'string' ? e : JSON.stringify(e, null, 2)
-        }
-    }
-}
-
-module.exports = { nodeClass: PythonInterpreter_Tools }
@@ -1 +0,0 @@
-<svg class="mr-1.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M15.84.5a16.4,16.4,0,0,0-3.57.32C9.1,1.39,8.53,2.53,8.53,4.64V7.48H16v1H5.77a4.73,4.73,0,0,0-4.7,3.74,14.82,14.82,0,0,0,0,7.54c.57,2.28,1.86,3.82,4,3.82h2.6V20.14a4.73,4.73,0,0,1,4.63-4.63h7.38a3.72,3.72,0,0,0,3.73-3.73V4.64A4.16,4.16,0,0,0,19.65.82,20.49,20.49,0,0,0,15.84.5ZM11.78,2.77a1.39,1.39,0,0,1,1.38,1.46,1.37,1.37,0,0,1-1.38,1.38A1.42,1.42,0,0,1,10.4,4.23,1.44,1.44,0,0,1,11.78,2.77Z" fill="#5a9fd4"></path><path d="M16.16,31.5a16.4,16.4,0,0,0,3.57-.32c3.17-.57,3.74-1.71,3.74-3.82V24.52H16v-1H26.23a4.73,4.73,0,0,0,4.7-3.74,14.82,14.82,0,0,0,0-7.54c-.57-2.28-1.86-3.82-4-3.82h-2.6v3.41a4.73,4.73,0,0,1-4.63,4.63H12.35a3.72,3.72,0,0,0-3.73,3.73v7.14a4.16,4.16,0,0,0,3.73,3.82A20.49,20.49,0,0,0,16.16,31.5Zm4.06-2.27a1.39,1.39,0,0,1-1.38-1.46,1.37,1.37,0,0,1,1.38-1.38,1.42,1.42,0,0,1,1.38,1.38A1.44,1.44,0,0,1,20.22,29.23Z" fill="#ffd43b"></path></svg>
				`@@ -1 +0,0 @@`
				<svg class="mr-1.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M15.84.5a16.4,16.4,0,0,0-3.57.32C9.1,1.39,8.53,2.53,8.53,4.64V7.48H16v1H5.77a4.73,4.73,0,0,0-4.7,3.74,14.82,14.82,0,0,0,0,7.54c.57,2.28,1.86,3.82,4,3.82h2.6V20.14a4.73,4.73,0,0,1,4.63-4.63h7.38a3.72,3.72,0,0,0,3.73-3.73V4.64A4.16,4.16,0,0,0,19.65.82,20.49,20.49,0,0,0,15.84.5ZM11.78,2.77a1.39,1.39,0,0,1,1.38,1.46,1.37,1.37,0,0,1-1.38,1.38A1.42,1.42,0,0,1,10.4,4.23,1.44,1.44,0,0,1,11.78,2.77Z" fill="#5a9fd4"></path><path d="M16.16,31.5a16.4,16.4,0,0,0,3.57-.32c3.17-.57,3.74-1.71,3.74-3.82V24.52H16v-1H26.23a4.73,4.73,0,0,0,4.7-3.74,14.82,14.82,0,0,0,0-7.54c-.57-2.28-1.86-3.82-4-3.82h-2.6v3.41a4.73,4.73,0,0,1-4.63,4.63H12.35a3.72,3.72,0,0,0-3.73,3.73v7.14a4.16,4.16,0,0,0,3.73,3.82A20.49,20.49,0,0,0,16.16,31.5Zm4.06-2.27a1.39,1.39,0,0,1-1.38-1.46,1.37,1.37,0,0,1,1.38-1.38,1.42,1.42,0,0,1,1.38,1.38A1.44,1.44,0,0,1,20.22,29.23Z" fill="#ffd43b"></path></svg>