Chore/refractor (#4454)

* markdown files and env examples cleanup

* components update

* update jsonlines description

* server refractor

* update telemetry

* add execute custom node

* add ui refractor

* add username and password authenticate

* correctly retrieve past images in agentflowv2

* disable e2e temporarily

* add existing username and password authenticate

* update migration to default workspace

* update todo

* blob storage migrating

* throw error on agent tool call error

* add missing execution import

* add referral

* chore: add error message when importData is undefined

* migrate api keys to db

* fix: data too long for column executionData

* migrate api keys from json to db at init

* add info on account setup

* update docstore missing fields

---------

Co-authored-by: chungyau97 <chungyau97@gmail.com>

packages/components/evaluation/EvaluationRunTracer.ts

@@ -0,0 +1,165 @@
+import { RunCollectorCallbackHandler } from '@langchain/core/tracers/run_collector'
+import { Run } from '@langchain/core/tracers/base'
+import { EvaluationRunner } from './EvaluationRunner'
+import { encoding_for_model, get_encoding } from '@dqbd/tiktoken'
+
+export class EvaluationRunTracer extends RunCollectorCallbackHandler {
+    evaluationRunId: string
+    model: string
+
+    constructor(id: string) {
+        super()
+        this.evaluationRunId = id
+    }
+
+    async persistRun(run: Run): Promise<void> {
+        return super.persistRun(run)
+    }
+
+    countPromptTokens = (encoding: any, run: Run): number => {
+        let promptTokenCount = 0
+        if (encoding) {
+            if (run.inputs?.messages?.length > 0 && run.inputs?.messages[0]?.length > 0) {
+                run.inputs.messages[0].map((message: any) => {
+                    let content = message.content
+                        ? message.content
+                        : message.SystemMessage?.content
+                        ? message.SystemMessage.content
+                        : message.HumanMessage?.content
+                        ? message.HumanMessage.content
+                        : message.AIMessage?.content
+                        ? message.AIMessage.content
+                        : undefined
+                    promptTokenCount += content ? encoding.encode(content).length : 0
+                })
+            }
+            if (run.inputs?.prompts?.length > 0) {
+                const content = run.inputs.prompts[0]
+                promptTokenCount += content ? encoding.encode(content).length : 0
+            }
+        }
+        return promptTokenCount
+    }
+
+    countCompletionTokens = (encoding: any, run: Run): number => {
+        let completionTokenCount = 0
+        if (encoding) {
+            if (run.outputs?.generations?.length > 0 && run.outputs?.generations[0]?.length > 0) {
+                run.outputs?.generations[0].map((chunk: any) => {
+                    let content = chunk.text ? chunk.text : chunk.message?.content ? chunk.message?.content : undefined
+                    completionTokenCount += content ? encoding.encode(content).length : 0
+                })
+            }
+        }
+        return completionTokenCount
+    }
+
+    extractModelName = (run: Run): string => {
+        return (
+            (run?.serialized as any)?.kwargs?.model ||
+            (run?.serialized as any)?.kwargs?.model_name ||
+            (run?.extra as any)?.metadata?.ls_model_name ||
+            (run?.extra as any)?.metadata?.fw_model_name
+        )
+    }
+
+    onLLMEnd?(run: Run): void | Promise<void> {
+        if (run.name) {
+            let provider = run.name
+            if (provider === 'BedrockChat') {
+                provider = 'awsChatBedrock'
+            }
+            EvaluationRunner.addMetrics(
+                this.evaluationRunId,
+                JSON.stringify({
+                    provider: provider
+                })
+            )
+        }
+
+        let model = this.extractModelName(run)
+        if (run.outputs?.llmOutput?.tokenUsage) {
+            const tokenUsage = run.outputs?.llmOutput?.tokenUsage
+            if (tokenUsage) {
+                const metric = {
+                    completionTokens: tokenUsage.completionTokens,
+                    promptTokens: tokenUsage.promptTokens,
+                    model: model,
+                    totalTokens: tokenUsage.totalTokens
+                }
+                EvaluationRunner.addMetrics(this.evaluationRunId, JSON.stringify(metric))
+            }
+        } else if (
+            run.outputs?.generations?.length > 0 &&
+            run.outputs?.generations[0].length > 0 &&
+            run.outputs?.generations[0][0]?.message?.usage_metadata?.total_tokens
+        ) {
+            const usage_metadata = run.outputs?.generations[0][0]?.message?.usage_metadata
+            if (usage_metadata) {
+                const metric = {
+                    completionTokens: usage_metadata.output_tokens,
+                    promptTokens: usage_metadata.input_tokens,
+                    model: model || this.model,
+                    totalTokens: usage_metadata.total_tokens
+                }
+                EvaluationRunner.addMetrics(this.evaluationRunId, JSON.stringify(metric))
+            }
+        } else {
+            let encoding: any = undefined
+            let promptInputTokens = 0
+            let completionTokenCount = 0
+            try {
+                encoding = encoding_for_model(model as any)
+                promptInputTokens = this.countPromptTokens(encoding, run)
+                completionTokenCount = this.countCompletionTokens(encoding, run)
+            } catch (e) {
+                try {
+                    // as tiktoken will fail for non openai models, assume that is 'cl100k_base'
+                    encoding = get_encoding('cl100k_base')
+                    promptInputTokens = this.countPromptTokens(encoding, run)
+                    completionTokenCount = this.countCompletionTokens(encoding, run)
+                } catch (e) {
+                    // stay silent
+                }
+            }
+            const metric = {
+                completionTokens: completionTokenCount,
+                promptTokens: promptInputTokens,
+                model: model,
+                totalTokens: promptInputTokens + completionTokenCount
+            }
+            EvaluationRunner.addMetrics(this.evaluationRunId, JSON.stringify(metric))
+            //cleanup
+            this.model = ''
+        }
+    }
+
+    async onRunUpdate(run: Run): Promise<void> {
+        const json = {
+            [run.run_type]: elapsed(run)
+        }
+        let metric = JSON.stringify(json)
+        if (metric) {
+            EvaluationRunner.addMetrics(this.evaluationRunId, metric)
+        }
+
+        if (run.run_type === 'llm') {
+            let model = this.extractModelName(run)
+            if (model) {
+                EvaluationRunner.addMetrics(this.evaluationRunId, JSON.stringify({ model: model }))
+                this.model = model
+            }
+            // OpenAI non streaming models
+            const estimatedTokenUsage = run.outputs?.llmOutput?.estimatedTokenUsage
+            if (estimatedTokenUsage && typeof estimatedTokenUsage === 'object' && Object.keys(estimatedTokenUsage).length > 0) {
+                EvaluationRunner.addMetrics(this.evaluationRunId, estimatedTokenUsage)
+            }
+        }
+    }
+}
+
+function elapsed(run: Run) {
+    if (!run.end_time) return ''
+    const elapsed = run.end_time - run.start_time
+    return `${elapsed.toFixed(2)}`
+}

packages/components/evaluation/EvaluationRunTracerLlama.ts

@@ -0,0 +1,186 @@
+import { ChatMessage, LLMEndEvent, LLMStartEvent, LLMStreamEvent, MessageContentTextDetail, RetrievalEndEvent, Settings } from 'llamaindex'
+import { EvaluationRunner } from './EvaluationRunner'
+import { additionalCallbacks, ICommonObject, INodeData } from '../src'
+import { RetrievalStartEvent } from 'llamaindex/dist/type/llm/types'
+import { AgentEndEvent, AgentStartEvent } from 'llamaindex/dist/type/agent/types'
+import { encoding_for_model } from '@dqbd/tiktoken'
+import { MessageContent } from '@langchain/core/messages'
+
+export class EvaluationRunTracerLlama {
+    evaluationRunId: string
+    static cbInit = false
+    static startTimes = new Map<string, number>()
+    static models = new Map<string, string>()
+    static tokenCounts = new Map<string, number>()
+
+    constructor(id: string) {
+        this.evaluationRunId = id
+        EvaluationRunTracerLlama.constructCallBacks()
+    }
+
+    static constructCallBacks = () => {
+        if (!EvaluationRunTracerLlama.cbInit) {
+            Settings.callbackManager.on('llm-start', (event: LLMStartEvent) => {
+                const evalID = (event as any).reason.parent?.caller?.evaluationRunId || (event as any).reason.caller?.evaluationRunId
+                if (!evalID) return
+                const model = (event as any).reason?.caller?.model
+                if (model) {
+                    EvaluationRunTracerLlama.models.set(evalID, model)
+                    try {
+                        const encoding = encoding_for_model(model)
+                        if (encoding) {
+                            const { messages } = event.detail.payload
+                            let tokenCount = messages.reduce((count: number, message: ChatMessage) => {
+                                return count + encoding.encode(extractText(message.content)).length
+                            }, 0)
+                            EvaluationRunTracerLlama.tokenCounts.set(evalID + '_promptTokens', tokenCount)
+                            EvaluationRunTracerLlama.tokenCounts.set(evalID + '_outputTokens', 0)
+                        }
+                    } catch (e) {
+                        // catch the error and continue to work.
+                    }
+                }
+                EvaluationRunTracerLlama.startTimes.set(evalID + '_llm', event.timeStamp)
+            })
+            Settings.callbackManager.on('llm-end', (event: LLMEndEvent) => {
+                this.calculateAndSetMetrics(event, 'llm')
+            })
+            Settings.callbackManager.on('llm-stream', (event: LLMStreamEvent) => {
+                const evalID = (event as any).reason.parent?.caller?.evaluationRunId || (event as any).reason.caller?.evaluationRunId
+                if (!evalID) return
+                const { chunk } = event.detail.payload
+                const { delta } = chunk
+                const model = (event as any).reason?.caller?.model
+                try {
+                    const encoding = encoding_for_model(model)
+                    if (encoding) {
+                        let tokenCount = EvaluationRunTracerLlama.tokenCounts.get(evalID + '_outputTokens') || 0
+                        tokenCount += encoding.encode(extractText(delta)).length
+                        EvaluationRunTracerLlama.tokenCounts.set(evalID + '_outputTokens', tokenCount)
+                    }
+                } catch (e) {
+                    // catch the error and continue to work.
+                }
+            })
+            Settings.callbackManager.on('retrieve-start', (event: RetrievalStartEvent) => {
+                const evalID = (event as any).reason.parent?.caller?.evaluationRunId || (event as any).reason.caller?.evaluationRunId
+                if (evalID) {
+                    EvaluationRunTracerLlama.startTimes.set(evalID + '_retriever', event.timeStamp)
+                }
+            })
+            Settings.callbackManager.on('retrieve-end', (event: RetrievalEndEvent) => {
+                this.calculateAndSetMetrics(event, 'retriever')
+            })
+            Settings.callbackManager.on('agent-start', (event: AgentStartEvent) => {
+                const evalID = (event as any).reason.parent?.caller?.evaluationRunId || (event as any).reason.caller?.evaluationRunId
+                if (evalID) {
+                    EvaluationRunTracerLlama.startTimes.set(evalID + '_agent', event.timeStamp)
+                }
+            })
+            Settings.callbackManager.on('agent-end', (event: AgentEndEvent) => {
+                this.calculateAndSetMetrics(event, 'agent')
+            })
+            EvaluationRunTracerLlama.cbInit = true
+        }
+    }
+
+    private static calculateAndSetMetrics(event: any, label: string) {
+        const evalID = event.reason.parent?.caller?.evaluationRunId || event.reason.caller?.evaluationRunId
+        if (!evalID) return
+        const startTime = EvaluationRunTracerLlama.startTimes.get(evalID + '_' + label) as number
+        let model =
+            (event as any).reason?.caller?.model || (event as any).reason?.caller?.llm?.model || EvaluationRunTracerLlama.models.get(evalID)
+
+        if (event.detail.payload?.response?.message && model) {
+            try {
+                const encoding = encoding_for_model(model)
+                if (encoding) {
+                    let tokenCount = EvaluationRunTracerLlama.tokenCounts.get(evalID + '_outputTokens') || 0
+                    tokenCount += encoding.encode(event.detail.payload.response?.message?.content || '').length
+                    EvaluationRunTracerLlama.tokenCounts.set(evalID + '_outputTokens', tokenCount)
+                }
+            } catch (e) {
+                // catch the error and continue to work.
+            }
+        }
+
+        // Anthropic
+        if (event.detail?.payload?.response?.raw?.usage) {
+            const usage = event.detail.payload.response.raw.usage
+            if (usage.output_tokens) {
+                const metric = {
+                    completionTokens: usage.output_tokens,
+                    promptTokens: usage.input_tokens,
+                    model: model,
+                    totalTokens: usage.input_tokens + usage.output_tokens
+                }
+                EvaluationRunner.addMetrics(evalID, JSON.stringify(metric))
+            } else if (usage.completion_tokens) {
+                const metric = {
+                    completionTokens: usage.completion_tokens,
+                    promptTokens: usage.prompt_tokens,
+                    model: model,
+                    totalTokens: usage.total_tokens
+                }
+                EvaluationRunner.addMetrics(evalID, JSON.stringify(metric))
+            }
+        } else if (event.detail?.payload?.response?.raw['amazon-bedrock-invocationMetrics']) {
+            const usage = event.detail?.payload?.response?.raw['amazon-bedrock-invocationMetrics']
+            const metric = {
+                completionTokens: usage.outputTokenCount,
+                promptTokens: usage.inputTokenCount,
+                model: event.detail?.payload?.response?.raw.model,
+                totalTokens: usage.inputTokenCount + usage.outputTokenCount
+            }
+            EvaluationRunner.addMetrics(evalID, JSON.stringify(metric))
+        } else {
+            const metric = {
+                [label]: (event.timeStamp - startTime).toFixed(2),
+                completionTokens: EvaluationRunTracerLlama.tokenCounts.get(evalID + '_outputTokens'),
+                promptTokens: EvaluationRunTracerLlama.tokenCounts.get(evalID + '_promptTokens'),
+                model: model || EvaluationRunTracerLlama.models.get(evalID) || '',
+                totalTokens:
+                    (EvaluationRunTracerLlama.tokenCounts.get(evalID + '_outputTokens') || 0) +
+                    (EvaluationRunTracerLlama.tokenCounts.get(evalID + '_promptTokens') || 0)
+            }
+            EvaluationRunner.addMetrics(evalID, JSON.stringify(metric))
+        }
+
+        //cleanup
+        EvaluationRunTracerLlama.startTimes.delete(evalID + '_' + label)
+        EvaluationRunTracerLlama.startTimes.delete(evalID + '_outputTokens')
+        EvaluationRunTracerLlama.startTimes.delete(evalID + '_promptTokens')
+        EvaluationRunTracerLlama.models.delete(evalID)
+    }
+
+    static async injectEvaluationMetadata(nodeData: INodeData, options: ICommonObject, callerObj: any) {
+        if (options.evaluationRunId && callerObj) {
+            // these are needed for evaluation runs
+            options.llamaIndex = true
+            await additionalCallbacks(nodeData, options)
+            Object.defineProperty(callerObj, 'evaluationRunId', {
+                enumerable: true,
+                configurable: true,
+                writable: true,
+                value: options.evaluationRunId
+            })
+        }
+    }
+}
+
+// from https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/llm/utils.ts
+export function extractText(message: MessageContent): string {
+    if (typeof message !== 'string' && !Array.isArray(message)) {
+        console.warn('extractText called with non-MessageContent message, this is likely a bug.')
+        return `${message}`
+    } else if (typeof message !== 'string' && Array.isArray(message)) {
+        // message is of type MessageContentDetail[] - retrieve just the text parts and concatenate them
+        // so we can pass them to the context generator
+        return message
+            .filter((c): c is MessageContentTextDetail => c.type === 'text')
+            .map((c) => c.text)
+            .join('\n\n')
+    } else {
+        return message
+    }
+}

packages/components/evaluation/EvaluationRunner.ts

@@ -0,0 +1,172 @@
+import axios from 'axios'
+import { v4 as uuidv4 } from 'uuid'
+import { ICommonObject } from '../src'
+
+import { getModelConfigByModelName, MODEL_TYPE } from '../src/modelLoader'
+
+export class EvaluationRunner {
+    static metrics = new Map<string, string[]>()
+    static async getAndDeleteMetrics(id: string) {
+        const val = EvaluationRunner.metrics.get(id)
+        if (val) {
+            try {
+                //first lets get the provider and model
+                let selectedModel = undefined
+                let selectedProvider = undefined
+                if (val && val.length > 0) {
+                    let modelName = ''
+                    let providerName = ''
+                    for (let i = 0; i < val.length; i++) {
+                        const metric = val[i]
+                        if (typeof metric === 'object') {
+                            modelName = metric['model']
+                            providerName = metric['provider']
+                        } else {
+                            modelName = JSON.parse(metric)['model']
+                            providerName = JSON.parse(metric)['provider']
+                        }
+
+                        if (modelName) {
+                            selectedModel = modelName
+                        }
+                        if (providerName) {
+                            selectedProvider = providerName
+                        }
+                    }
+                }
+                let modelConfig = await getModelConfigByModelName(MODEL_TYPE.CHAT, selectedProvider, selectedModel)
+                if (modelConfig) {
+                    val.push(JSON.stringify({ cost_values: modelConfig }))
+                } else {
+                    modelConfig = await getModelConfigByModelName(MODEL_TYPE.LLM, selectedProvider, selectedModel)
+                    if (modelConfig) {
+                        val.push(JSON.stringify({ cost_values: modelConfig }))
+                    }
+                }
+            } catch (error) {
+                //stay silent
+            }
+        }
+        EvaluationRunner.metrics.delete(id)
+        return val
+    }
+
+    static addMetrics(id: string, metric: string) {
+        if (EvaluationRunner.metrics.has(id)) {
+            EvaluationRunner.metrics.get(id)?.push(metric)
+        } else {
+            EvaluationRunner.metrics.set(id, [metric])
+        }
+    }
+
+    baseURL = ''
+
+    constructor(baseURL: string) {
+        this.baseURL = baseURL
+    }
+
+    getChatflowApiKey(chatflowId: string, apiKeys: { chatflowId: string; apiKey: string }[] = []) {
+        return apiKeys.find((item) => item.chatflowId === chatflowId)?.apiKey || ''
+    }
+
+    public async runEvaluations(data: ICommonObject) {
+        const chatflowIds = JSON.parse(data.chatflowId)
+        const returnData: ICommonObject = {}
+        returnData.evaluationId = data.evaluationId
+        returnData.runDate = new Date()
+        returnData.rows = []
+        for (let i = 0; i < data.dataset.rows.length; i++) {
+            returnData.rows.push({
+                input: data.dataset.rows[i].input,
+                expectedOutput: data.dataset.rows[i].output,
+                itemNo: data.dataset.rows[i].sequenceNo,
+                evaluations: [],
+                status: 'pending'
+            })
+        }
+        for (let i = 0; i < chatflowIds.length; i++) {
+            const chatflowId = chatflowIds[i]
+            await this.evaluateChatflow(chatflowId, this.getChatflowApiKey(chatflowId, data.apiKeys), data, returnData)
+        }
+        return returnData
+    }
+
+    async evaluateChatflow(chatflowId: string, apiKey: string, data: any, returnData: any) {
+        for (let i = 0; i < data.dataset.rows.length; i++) {
+            const item = data.dataset.rows[i]
+            const uuid = uuidv4()
+
+            const headers: any = {
+                'X-Request-ID': uuid,
+                'X-Flowise-Evaluation': 'true'
+            }
+            if (apiKey) {
+                headers['Authorization'] = `Bearer ${apiKey}`
+            }
+            let axiosConfig = {
+                headers: headers
+            }
+            let startTime = performance.now()
+            const runData: any = {}
+            runData.chatflowId = chatflowId
+            runData.startTime = startTime
+            const postData: any = { question: item.input, evaluationRunId: uuid, evaluation: true }
+            if (data.sessionId) {
+                postData.overrideConfig = { sessionId: data.sessionId }
+            }
+            try {
+                let response = await axios.post(`${this.baseURL}/api/v1/prediction/${chatflowId}`, postData, axiosConfig)
+                const endTime = performance.now()
+                const timeTaken = (endTime - startTime).toFixed(2)
+                if (response?.data?.metrics) {
+                    runData.metrics = response.data.metrics
+                    runData.metrics.push({
+                        apiLatency: timeTaken
+                    })
+                } else {
+                    runData.metrics = [
+                        {
+                            apiLatency: timeTaken
+                        }
+                    ]
+                }
+                runData.status = 'complete'
+                let resultText = ''
+                if (response.data.text) resultText = response.data.text
+                else if (response.data.json) resultText = '```json\n' + JSON.stringify(response.data.json, null, 2)
+                else resultText = JSON.stringify(response.data, null, 2)
+
+                runData.actualOutput = resultText
+                runData.latency = timeTaken
+                runData.error = ''
+            } catch (error: any) {
+                runData.status = 'error'
+                runData.actualOutput = ''
+                runData.error = error?.response?.data?.message
+                    ? error.response.data.message
+                    : error?.message
+                    ? error.message
+                    : 'Unknown error'
+                try {
+                    if (runData.error.indexOf('-') > -1) {
+                        // if there is a dash, remove all content before
+                        runData.error = 'Error: ' + runData.error.substr(runData.error.indexOf('-') + 1).trim()
+                    }
+                } catch (error) {
+                    //stay silent
+                }
+                const endTime = performance.now()
+                const timeTaken = (endTime - startTime).toFixed(2)
+                runData.metrics = [
+                    {
+                        apiLatency: timeTaken
+                    }
+                ]
+                runData.latency = timeTaken
+            }
+            runData.uuid = uuid
+            returnData.rows[i].evaluations.push(runData)
+        }
+        return returnData
+    }
+}

packages/components/nodes/agentflow/Agent/Agent.ts

@@ -427,7 +427,8 @@ class Agent_Agentflow implements INode {
                 return returnData
             }
 
-            const stores = await appDataSource.getRepository(databaseEntities['DocumentStore']).find()
+            const searchOptions = options.searchOptions || {}
+            const stores = await appDataSource.getRepository(databaseEntities['DocumentStore']).findBy(searchOptions)
             for (const store of stores) {
                 if (store.status === 'UPSERTED') {
                     const obj = {

packages/components/nodes/agentflow/CustomFunction/CustomFunction.ts

@@ -152,7 +152,7 @@ class CustomFunction_Agentflow implements INode {
             newState = updateFlowState(state, _customFunctionUpdateState)
         }
 
-        const variables = await getVars(appDataSource, databaseEntities, nodeData)
+        const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
         const flow = {
             chatflowId: options.chatflowid,
             sessionId: options.sessionId,

packages/components/nodes/agentflow/ExecuteFlow/ExecuteFlow.ts

@@ -127,7 +127,8 @@ class ExecuteFlow_Agentflow implements INode {
                 return returnData
             }
 
-            const chatflows = await appDataSource.getRepository(databaseEntities['ChatFlow']).find()
+            const searchOptions = options.searchOptions || {}
+            const chatflows = await appDataSource.getRepository(databaseEntities['ChatFlow']).findBy(searchOptions)
 
             for (let i = 0; i < chatflows.length; i += 1) {
                 let cfType = 'Chatflow'

packages/components/nodes/agentflow/Retriever/Retriever.ts

@@ -119,7 +119,8 @@ class Retriever_Agentflow implements INode {
                 return returnData
             }
 
-            const stores = await appDataSource.getRepository(databaseEntities['DocumentStore']).find()
+            const searchOptions = options.searchOptions || {}
+            const stores = await appDataSource.getRepository(databaseEntities['DocumentStore']).findBy(searchOptions)
             for (const store of stores) {
                 if (store.status === 'UPSERTED') {
                     const obj = {

packages/components/nodes/agentflow/utils.ts

@@ -18,7 +18,7 @@ export const addImagesToMessages = async (
         for (const upload of imageUploads) {
             let bf = upload.data
             if (upload.type == 'stored-file') {
-                const contents = await getFileFromStorage(upload.name, options.chatflowid, options.chatId)
+                const contents = await getFileFromStorage(upload.name, options.orgId, options.chatflowid, options.chatId)
                 // as the image is stored in the server, read the file and convert it to base64
                 bf = 'data:' + upload.mime + ';base64,' + contents.toString('base64')
 
@@ -90,7 +90,7 @@ export const processMessagesWithImages = async (
                     hasImageReferences = true
                     try {
                         // Get file contents from storage
-                        const contents = await getFileFromStorage(item.name, options.chatflowid, options.chatId)
+                        const contents = await getFileFromStorage(item.name, options.orgId, options.chatflowid, options.chatId)
 
                         // Create base64 data URL
                         const base64Data = 'data:' + item.mime + ';base64,' + contents.toString('base64')
@@ -319,7 +319,7 @@ export const getPastChatHistoryImageMessages = async (
                 const imageContents: MessageContentImageUrl[] = []
                 for (const upload of uploads) {
                     if (upload.type === 'stored-file' && upload.mime.startsWith('image/')) {
-                        const fileData = await getFileFromStorage(upload.name, options.chatflowid, options.chatId)
+                        const fileData = await getFileFromStorage(upload.name, options.orgId, options.chatflowid, options.chatId)
                         // as the image is stored in the server, read the file and convert it to base64
                         const bf = 'data:' + upload.mime + ';base64,' + fileData.toString('base64')
 

packages/components/nodes/agents/AirtableAgent/AirtableAgent.ts

@@ -128,7 +128,7 @@ class Airtable_Agents implements INode {
 
         let base64String = Buffer.from(JSON.stringify(airtableData)).toString('base64')
 
-        const loggerHandler = new ConsoleCallbackHandler(options.logger)
+        const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
         const callbacks = await additionalCallbacks(nodeData, options)
 
         const pyodide = await LoadPyodide()
@@ -163,7 +163,7 @@ json.dumps(my_dict)`
             const chain = new LLMChain({
                 llm: model,
                 prompt: PromptTemplate.fromTemplate(systemPrompt),
-                verbose: process.env.DEBUG === 'true'
+                verbose: process.env.DEBUG === 'true' ? true : false
             })
             const inputs = {
                 dict: dataframeColDict,
@@ -192,7 +192,7 @@ json.dumps(my_dict)`
             const chain = new LLMChain({
                 llm: model,
                 prompt: PromptTemplate.fromTemplate(finalSystemPrompt),
-                verbose: process.env.DEBUG === 'true'
+                verbose: process.env.DEBUG === 'true' ? true : false
             })
             const inputs = {
                 question: input,

packages/components/nodes/agents/CSVAgent/CSVAgent.ts

@@ -97,7 +97,7 @@ class CSV_Agents implements INode {
             }
         }
 
-        const loggerHandler = new ConsoleCallbackHandler(options.logger)
+        const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
         const shouldStreamResponse = options.shouldStreamResponse
         const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
         const chatId = options.chatId
@@ -114,11 +114,12 @@ class CSV_Agents implements INode {
             } else {
                 files = [fileName]
             }
+            const orgId = options.orgId
             const chatflowid = options.chatflowid
 
             for (const file of files) {
                 if (!file) continue
-                const fileData = await getFileFromStorage(file, chatflowid)
+                const fileData = await getFileFromStorage(file, orgId, chatflowid)
                 base64String += fileData.toString('base64')
             }
         } else {
@@ -170,7 +171,7 @@ json.dumps(my_dict)`
             const chain = new LLMChain({
                 llm: model,
                 prompt: PromptTemplate.fromTemplate(systemPrompt),
-                verbose: process.env.DEBUG === 'true'
+                verbose: process.env.DEBUG === 'true' ? true : false
             })
             const inputs = {
                 dict: dataframeColDict,
@@ -201,7 +202,7 @@ json.dumps(my_dict)`
                 prompt: PromptTemplate.fromTemplate(
                     systemMessagePrompt ? `${systemMessagePrompt}\n${finalSystemPrompt}` : finalSystemPrompt
                 ),
-                verbose: process.env.DEBUG === 'true'
+                verbose: process.env.DEBUG === 'true' ? true : false
             })
             const inputs = {
                 question: input,

packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts

@@ -132,7 +132,7 @@ class ConversationalAgent_Agents implements INode {
         }
         const executor = await prepareAgent(nodeData, options, { sessionId: this.sessionId, chatId: options.chatId, input })
 
-        const loggerHandler = new ConsoleCallbackHandler(options.logger)
+        const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
         const callbacks = await additionalCallbacks(nodeData, options)
 
         let res: ChainValues = {}

packages/components/nodes/agents/ConversationalRetrievalToolAgent/ConversationalRetrievalToolAgent.ts

@@ -130,7 +130,7 @@ class ConversationalRetrievalToolAgent_Agents implements INode {
 
         const executor = await prepareAgent(nodeData, options, { sessionId: this.sessionId, chatId: options.chatId, input })
 
-        const loggerHandler = new ConsoleCallbackHandler(options.logger)
+        const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
         const callbacks = await additionalCallbacks(nodeData, options)
 
         let res: ChainValues = {}
@@ -288,7 +288,7 @@ const prepareAgent = async (
         sessionId: flowObj?.sessionId,
         chatId: flowObj?.chatId,
         input: flowObj?.input,
-        verbose: process.env.DEBUG === 'true',
+        verbose: process.env.DEBUG === 'true' ? true : false,
         maxIterations: maxIterations ? parseFloat(maxIterations) : undefined
     })
 

packages/components/nodes/agents/LlamaIndexAgents/AnthropicAgent/AnthropicAgent_LlamaIndex.ts

@@ -2,6 +2,7 @@ import { flatten } from 'lodash'
 import { MessageContentTextDetail, ChatMessage, AnthropicAgent, Anthropic } from 'llamaindex'
 import { getBaseClasses } from '../../../../src/utils'
 import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams, IUsedTool } from '../../../../src/Interface'
+import { EvaluationRunTracerLlama } from '../../../../evaluation/EvaluationRunTracerLlama'
 
 class AnthropicAgent_LlamaIndex_Agents implements INode {
     label: string
@@ -96,13 +97,16 @@ class AnthropicAgent_LlamaIndex_Agents implements INode {
             tools,
             llm: model,
             chatHistory: chatHistory,
-            verbose: process.env.DEBUG === 'true'
+            verbose: process.env.DEBUG === 'true' ? true : false
         })
 
+        // these are needed for evaluation runs
+        await EvaluationRunTracerLlama.injectEvaluationMetadata(nodeData, options, agent)
+
         let text = ''
         const usedTools: IUsedTool[] = []
 
-        const response = await agent.chat({ message: input, chatHistory, verbose: process.env.DEBUG === 'true' })
+        const response = await agent.chat({ message: input, chatHistory, verbose: process.env.DEBUG === 'true' ? true : false })
 
         if (response.sources.length) {
             for (const sourceTool of response.sources) {

packages/components/nodes/agents/LlamaIndexAgents/OpenAIToolAgent/OpenAIToolAgent_LlamaIndex.ts

@@ -1,6 +1,7 @@
 import { flatten } from 'lodash'
 import { ChatMessage, OpenAI, OpenAIAgent } from 'llamaindex'
 import { getBaseClasses } from '../../../../src/utils'
+import { EvaluationRunTracerLlama } from '../../../../evaluation/EvaluationRunTracerLlama'
 import {
     FlowiseMemory,
     ICommonObject,
@@ -107,9 +108,12 @@ class OpenAIFunctionAgent_LlamaIndex_Agents implements INode {
             tools,
             llm: model,
             chatHistory: chatHistory,
-            verbose: process.env.DEBUG === 'true'
+            verbose: process.env.DEBUG === 'true' ? true : false
         })
 
+        // these are needed for evaluation runs
+        await EvaluationRunTracerLlama.injectEvaluationMetadata(nodeData, options, agent)
+
         let text = ''
         let isStreamingStarted = false
         const usedTools: IUsedTool[] = []
@@ -119,10 +123,9 @@ class OpenAIFunctionAgent_LlamaIndex_Agents implements INode {
                 message: input,
                 chatHistory,
                 stream: true,
-                verbose: process.env.DEBUG === 'true'
+                verbose: process.env.DEBUG === 'true' ? true : false
             })
             for await (const chunk of stream) {
-                //console.log('chunk', chunk)
                 text += chunk.response.delta
                 if (!isStreamingStarted) {
                     isStreamingStarted = true
@@ -147,7 +150,7 @@ class OpenAIFunctionAgent_LlamaIndex_Agents implements INode {
                 }
             }
         } else {
-            const response = await agent.chat({ message: input, chatHistory, verbose: process.env.DEBUG === 'true' })
+            const response = await agent.chat({ message: input, chatHistory, verbose: process.env.DEBUG === 'true' ? true : false })
             if (response.sources.length) {
                 for (const sourceTool of response.sources) {
                     usedTools.push({

packages/components/nodes/agents/OpenAIAssistant/OpenAIAssistant.ts

@@ -107,7 +107,11 @@ class OpenAIAssistant_Agents implements INode {
                 return returnData
             }
 
-            const assistants = await appDataSource.getRepository(databaseEntities['Assistant']).find()
+            const searchOptions = options.searchOptions || {}
+            const assistants = await appDataSource.getRepository(databaseEntities['Assistant']).findBy({
+                ...searchOptions,
+                type: 'OPENAI'
+            })
 
             for (let i = 0; i < assistants.length; i += 1) {
                 const assistantDetails = JSON.parse(assistants[i].details)
@@ -130,13 +134,14 @@ class OpenAIAssistant_Agents implements INode {
         const selectedAssistantId = nodeData.inputs?.selectedAssistant as string
         const appDataSource = options.appDataSource as DataSource
         const databaseEntities = options.databaseEntities as IDatabaseEntity
+        const orgId = options.orgId
 
         const assistant = await appDataSource.getRepository(databaseEntities['Assistant']).findOneBy({
             id: selectedAssistantId
         })
 
         if (!assistant) {
-            options.logger.error(`Assistant ${selectedAssistantId} not found`)
+            options.logger.error(`[${orgId}]: Assistant ${selectedAssistantId} not found`)
             return
         }
 
@@ -149,7 +154,7 @@ class OpenAIAssistant_Agents implements INode {
                 chatId
             })
             if (!chatmsg) {
-                options.logger.error(`Chat Message with Chat Id: ${chatId} not found`)
+                options.logger.error(`[${orgId}]: Chat Message with Chat Id: ${chatId} not found`)
                 return
             }
             sessionId = chatmsg.sessionId
@@ -160,21 +165,21 @@ class OpenAIAssistant_Agents implements INode {
         const credentialData = await getCredentialData(assistant.credential ?? '', options)
         const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData)
         if (!openAIApiKey) {
-            options.logger.error(`OpenAI ApiKey not found`)
+            options.logger.error(`[${orgId}]: OpenAI ApiKey not found`)
             return
         }
 
         const openai = new OpenAI({ apiKey: openAIApiKey })
-        options.logger.info(`Clearing OpenAI Thread ${sessionId}`)
+        options.logger.info(`[${orgId}]: Clearing OpenAI Thread ${sessionId}`)
         try {
             if (sessionId && sessionId.startsWith('thread_')) {
                 await openai.beta.threads.del(sessionId)
-                options.logger.info(`Successfully cleared OpenAI Thread ${sessionId}`)
+                options.logger.info(`[${orgId}]: Successfully cleared OpenAI Thread ${sessionId}`)
             } else {
-                options.logger.error(`Error clearing OpenAI Thread ${sessionId}`)
+                options.logger.error(`[${orgId}]: Error clearing OpenAI Thread ${sessionId}`)
             }
         } catch (e) {
-            options.logger.error(`Error clearing OpenAI Thread ${sessionId}`)
+            options.logger.error(`[${orgId}]: Error clearing OpenAI Thread ${sessionId}`)
         }
     }
 
@@ -190,6 +195,17 @@ class OpenAIAssistant_Agents implements INode {
         const shouldStreamResponse = options.shouldStreamResponse
         const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
         const chatId = options.chatId
+        const checkStorage = options.checkStorage
+            ? (options.checkStorage as (orgId: string, subscriptionId: string, usageCacheManager: any) => Promise<void>)
+            : undefined
+        const updateStorageUsage = options.updateStorageUsage
+            ? (options.updateStorageUsage as (
+                  orgId: string,
+                  workspaceId: string,
+                  totalSize: number,
+                  usageCacheManager: any
+              ) => Promise<void>)
+            : undefined
 
         if (moderations && moderations.length > 0) {
             try {
@@ -380,17 +396,30 @@ class OpenAIAssistant_Agents implements INode {
                                         // eslint-disable-next-line no-useless-escape
                                         const fileName = cited_file.filename.split(/[\/\\]/).pop() ?? cited_file.filename
                                         if (!disableFileDownload) {
-                                            filePath = await downloadFile(
+                                            if (checkStorage)
+                                                await checkStorage(options.orgId, options.subscriptionId, options.usageCacheManager)
+
+                                            const { path, totalSize } = await downloadFile(
                                                 openAIApiKey,
                                                 cited_file,
                                                 fileName,
+                                                options.orgId,
                                                 options.chatflowid,
                                                 options.chatId
                                             )
+                                            filePath = path
                                             fileAnnotations.push({
                                                 filePath,
                                                 fileName
                                             })
+
+                                            if (updateStorageUsage)
+                                                await updateStorageUsage(
+                                                    options.orgId,
+                                                    options.workspaceId,
+                                                    totalSize,
+                                                    options.usageCacheManager
+                                                )
                                         }
                                     } else {
                                         const file_path = (annotation as OpenAI.Beta.Threads.Messages.FilePathAnnotation).file_path
@@ -399,17 +428,30 @@ class OpenAIAssistant_Agents implements INode {
                                             // eslint-disable-next-line no-useless-escape
                                             const fileName = cited_file.filename.split(/[\/\\]/).pop() ?? cited_file.filename
                                             if (!disableFileDownload) {
-                                                filePath = await downloadFile(
+                                                if (checkStorage)
+                                                    await checkStorage(options.orgId, options.subscriptionId, options.usageCacheManager)
+
+                                                const { path, totalSize } = await downloadFile(
                                                     openAIApiKey,
                                                     cited_file,
                                                     fileName,
+                                                    options.orgId,
                                                     options.chatflowid,
                                                     options.chatId
                                                 )
+                                                filePath = path
                                                 fileAnnotations.push({
                                                     filePath,
                                                     fileName
                                                 })
+
+                                                if (updateStorageUsage)
+                                                    await updateStorageUsage(
+                                                        options.orgId,
+                                                        options.workspaceId,
+                                                        totalSize,
+                                                        options.usageCacheManager
+                                                    )
                                             }
                                         }
                                     }
@@ -467,15 +509,21 @@ class OpenAIAssistant_Agents implements INode {
                             const fileId = chunk.image_file.file_id
                             const fileObj = await openai.files.retrieve(fileId)
 
-                            const filePath = await downloadImg(
+                            if (checkStorage) await checkStorage(options.orgId, options.subscriptionId, options.usageCacheManager)
+
+                            const { filePath, totalSize } = await downloadImg(
                                 openai,
                                 fileId,
                                 `${fileObj.filename}.png`,
+                                options.orgId,
                                 options.chatflowid,
                                 options.chatId
                             )
                             artifacts.push({ type: 'png', data: filePath })
 
+                            if (updateStorageUsage)
+                                await updateStorageUsage(options.orgId, options.workspaceId, totalSize, options.usageCacheManager)
+
                             if (!isStreamingStarted) {
                                 isStreamingStarted = true
                                 if (sseStreamer) {
@@ -776,7 +824,21 @@ class OpenAIAssistant_Agents implements INode {
                                 // eslint-disable-next-line no-useless-escape
                                 const fileName = cited_file.filename.split(/[\/\\]/).pop() ?? cited_file.filename
                                 if (!disableFileDownload) {
-                                    filePath = await downloadFile(openAIApiKey, cited_file, fileName, options.chatflowid, options.chatId)
+                                    if (checkStorage) await checkStorage(options.orgId, options.subscriptionId, options.usageCacheManager)
+
+                                    const { path, totalSize } = await downloadFile(
+                                        openAIApiKey,
+                                        cited_file,
+                                        fileName,
+                                        options.orgId,
+                                        options.chatflowid,
+                                        options.chatId
+                                    )
+                                    filePath = path
+
+                                    if (updateStorageUsage)
+                                        await updateStorageUsage(options.orgId, options.workspaceId, totalSize, options.usageCacheManager)
+
                                     fileAnnotations.push({
                                         filePath,
                                         fileName
@@ -789,13 +851,27 @@ class OpenAIAssistant_Agents implements INode {
                                     // eslint-disable-next-line no-useless-escape
                                     const fileName = cited_file.filename.split(/[\/\\]/).pop() ?? cited_file.filename
                                     if (!disableFileDownload) {
-                                        filePath = await downloadFile(
+                                        if (checkStorage)
+                                            await checkStorage(options.orgId, options.subscriptionId, options.usageCacheManager)
+
+                                        const { path, totalSize } = await downloadFile(
                                             openAIApiKey,
                                             cited_file,
                                             fileName,
+                                            options.orgId,
                                             options.chatflowid,
                                             options.chatId
                                         )
+                                        filePath = path
+
+                                        if (updateStorageUsage)
+                                            await updateStorageUsage(
+                                                options.orgId,
+                                                options.workspaceId,
+                                                totalSize,
+                                                options.usageCacheManager
+                                            )
+
                                         fileAnnotations.push({
                                             filePath,
                                             fileName
@@ -822,7 +898,20 @@ class OpenAIAssistant_Agents implements INode {
                     const fileId = content.image_file.file_id
                     const fileObj = await openai.files.retrieve(fileId)
 
-                    const filePath = await downloadImg(openai, fileId, `${fileObj.filename}.png`, options.chatflowid, options.chatId)
+                    if (checkStorage) await checkStorage(options.orgId, options.subscriptionId, options.usageCacheManager)
+
+                    const { filePath, totalSize } = await downloadImg(
+                        openai,
+                        fileId,
+                        `${fileObj.filename}.png`,
+                        options.orgId,
+                        options.chatflowid,
+                        options.chatId
+                    )
+
+                    if (updateStorageUsage)
+                        await updateStorageUsage(options.orgId, options.workspaceId, totalSize, options.usageCacheManager)
+
                     artifacts.push({ type: 'png', data: filePath })
                 }
             }
@@ -847,7 +936,13 @@ class OpenAIAssistant_Agents implements INode {
     }
 }
 
-const downloadImg = async (openai: OpenAI, fileId: string, fileName: string, ...paths: string[]) => {
+const downloadImg = async (
+    openai: OpenAI,
+    fileId: string,
+    fileName: string,
+    orgId: string,
+    ...paths: string[]
+): Promise<{ filePath: string; totalSize: number }> => {
     const response = await openai.files.content(fileId)
 
     // Extract the binary data from the Response object
@@ -857,12 +952,18 @@ const downloadImg = async (openai: OpenAI, fileId: string, fileName: string, ...
     const image_data_buffer = Buffer.from(image_data)
     const mime = 'image/png'
 
-    const res = await addSingleFileToStorage(mime, image_data_buffer, fileName, ...paths)
+    const { path, totalSize } = await addSingleFileToStorage(mime, image_data_buffer, fileName, orgId, ...paths)
 
-    return res
+    return { filePath: path, totalSize }
 }
 
-const downloadFile = async (openAIApiKey: string, fileObj: any, fileName: string, ...paths: string[]) => {
+const downloadFile = async (
+    openAIApiKey: string,
+    fileObj: any,
+    fileName: string,
+    orgId: string,
+    ...paths: string[]
+): Promise<{ path: string; totalSize: number }> => {
     try {
         const response = await fetch(`https://api.openai.com/v1/files/${fileObj.id}/content`, {
             method: 'GET',
@@ -880,10 +981,12 @@ const downloadFile = async (openAIApiKey: string, fileObj: any, fileName: string
         const data_buffer = Buffer.from(data)
         const mime = 'application/octet-stream'
 
-        return await addSingleFileToStorage(mime, data_buffer, fileName, ...paths)
+        const { path, totalSize } = await addSingleFileToStorage(mime, data_buffer, fileName, orgId, ...paths)
+
+        return { path, totalSize }
     } catch (error) {
         console.error('Error downloading or writing the file:', error)
-        return ''
+        return { path: '', totalSize: 0 }
     }
 }
 

packages/components/nodes/agents/ReActAgentLLM/ReActAgentLLM.ts

@@ -97,7 +97,7 @@ class ReActAgentLLM_Agents implements INode {
         const executor = new AgentExecutor({
             agent,
             tools,
-            verbose: process.env.DEBUG === 'true',
+            verbose: process.env.DEBUG === 'true' ? true : false,
             maxIterations: maxIterations ? parseFloat(maxIterations) : undefined
         })
 

packages/components/nodes/agents/ToolAgent/ToolAgent.ts

@@ -143,7 +143,7 @@ class ToolAgent_Agents implements INode {
 
         const executor = await prepareAgent(nodeData, options, { sessionId: this.sessionId, chatId: options.chatId, input })
 
-        const loggerHandler = new ConsoleCallbackHandler(options.logger)
+        const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
         const callbacks = await additionalCallbacks(nodeData, options)
 
         // Add custom streaming handler if detailed streaming is enabled
@@ -370,7 +370,7 @@ const prepareAgent = async (
         sessionId: flowObj?.sessionId,
         chatId: flowObj?.chatId,
         input: flowObj?.input,
-        verbose: process.env.DEBUG === 'true',
+        verbose: process.env.DEBUG === 'true' ? true : false,
         maxIterations: maxIterations ? parseFloat(maxIterations) : undefined
     })
 

packages/components/nodes/agents/XMLAgent/XMLAgent.ts

@@ -138,7 +138,7 @@ class XMLAgent_Agents implements INode {
         }
         const executor = await prepareAgent(nodeData, options, { sessionId: this.sessionId, chatId: options.chatId, input })
 
-        const loggerHandler = new ConsoleCallbackHandler(options.logger)
+        const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
         const callbacks = await additionalCallbacks(nodeData, options)
 
         let res: ChainValues = {}
@@ -278,7 +278,7 @@ const prepareAgent = async (
         chatId: flowObj?.chatId,
         input: flowObj?.input,
         isXML: true,
-        verbose: process.env.DEBUG === 'true',
+        verbose: process.env.DEBUG === 'true' ? true : false,
         maxIterations: maxIterations ? parseFloat(maxIterations) : undefined
     })
 

packages/components/nodes/chains/ApiChain/GETApiChain.ts

@@ -98,7 +98,7 @@ class GETApiChain_Chains implements INode {
         const ansPrompt = nodeData.inputs?.ansPrompt as string
 
         const chain = await getAPIChain(apiDocs, model, headers, urlPrompt, ansPrompt)
-        const loggerHandler = new ConsoleCallbackHandler(options.logger)
+        const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
         const callbacks = await additionalCallbacks(nodeData, options)
         const shouldStreamResponse = options.shouldStreamResponse
         const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
@@ -129,7 +129,7 @@ const getAPIChain = async (documents: string, llm: BaseLanguageModel, headers: s
     const chain = APIChain.fromLLMAndAPIDocs(llm, documents, {
         apiUrlPrompt,
         apiResponsePrompt,
-        verbose: process.env.DEBUG === 'true',
+        verbose: process.env.DEBUG === 'true' ? true : false,
         headers: typeof headers === 'object' ? headers : headers ? JSON.parse(headers) : {}
     })
     return chain

packages/components/nodes/chains/ApiChain/OpenAPIChain.ts

@@ -71,7 +71,7 @@ class OpenApiChain_Chains implements INode {
 
     async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string | object> {
         const chain = await initChain(nodeData, options)
-        const loggerHandler = new ConsoleCallbackHandler(options.logger)
+        const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
         const callbacks = await additionalCallbacks(nodeData, options)
         const moderations = nodeData.inputs?.inputModeration as Moderation[]
         const shouldStreamResponse = options.shouldStreamResponse
@@ -114,8 +114,9 @@ const initChain = async (nodeData: INodeData, options: ICommonObject) => {
     } else {
         if (yamlFileBase64.startsWith('FILE-STORAGE::')) {
             const file = yamlFileBase64.replace('FILE-STORAGE::', '')
+            const orgId = options.orgId
             const chatflowid = options.chatflowid
-            const fileData = await getFileFromStorage(file, chatflowid)
+            const fileData = await getFileFromStorage(file, orgId, chatflowid)
             yamlString = fileData.toString()
         } else {
             const splitDataURI = yamlFileBase64.split(',')
@@ -128,7 +129,7 @@ const initChain = async (nodeData: INodeData, options: ICommonObject) => {
     return await createOpenAPIChain(yamlString, {
         llm: model,
         headers: typeof headers === 'object' ? headers : headers ? JSON.parse(headers) : {},
-        verbose: process.env.DEBUG === 'true'
+        verbose: process.env.DEBUG === 'true' ? true : false
     })
 }
 

packages/components/nodes/chains/ApiChain/POSTApiChain.ts

@@ -87,7 +87,7 @@ class POSTApiChain_Chains implements INode {
         const ansPrompt = nodeData.inputs?.ansPrompt as string
 
         const chain = await getAPIChain(apiDocs, model, headers, urlPrompt, ansPrompt)
-        const loggerHandler = new ConsoleCallbackHandler(options.logger)
+        const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
         const callbacks = await additionalCallbacks(nodeData, options)
 
         const shouldStreamResponse = options.shouldStreamResponse
@@ -119,7 +119,7 @@ const getAPIChain = async (documents: string, llm: BaseLanguageModel, headers: s
     const chain = APIChain.fromLLMAndAPIDocs(llm, documents, {
         apiUrlPrompt,
         apiResponsePrompt,
-        verbose: process.env.DEBUG === 'true',
+        verbose: process.env.DEBUG === 'true' ? true : false,
         headers: typeof headers === 'object' ? headers : headers ? JSON.parse(headers) : {}
     })
     return chain

packages/components/nodes/chains/ConversationChain/ConversationChain.ts

@@ -132,7 +132,7 @@ class ConversationChain_Chains implements INode {
             }
         }
 
-        const loggerHandler = new ConsoleCallbackHandler(options.logger)
+        const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
         const additionalCallback = await additionalCallbacks(nodeData, options)
 
         let res = ''

packages/components/nodes/chains/ConversationalRetrievalQAChain/ConversationalRetrievalQAChain.ts

@@ -185,6 +185,7 @@ class ConversationalRetrievalQAChain_Chains implements INode {
         const shouldStreamResponse = options.shouldStreamResponse
         const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
         const chatId = options.chatId
+        const orgId = options.orgId
 
         let customResponsePrompt = responsePrompt
         // If the deprecated systemMessagePrompt is still exists
@@ -200,7 +201,8 @@ class ConversationalRetrievalQAChain_Chains implements INode {
                 memoryKey: 'chat_history',
                 appDataSource,
                 databaseEntities,
-                chatflowid
+                chatflowid,
+                orgId
             })
         }
 
@@ -220,7 +222,7 @@ class ConversationalRetrievalQAChain_Chains implements INode {
 
         const history = ((await memory.getChatMessages(this.sessionId, false, prependMessages)) as IMessage[]) ?? []
 
-        const loggerHandler = new ConsoleCallbackHandler(options.logger)
+        const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
         const additionalCallback = await additionalCallbacks(nodeData, options)
 
         let callbacks = [loggerHandler, ...additionalCallback]
@@ -407,18 +409,21 @@ interface BufferMemoryExtendedInput {
     appDataSource: DataSource
     databaseEntities: IDatabaseEntity
     chatflowid: string
+    orgId: string
 }
 
 class BufferMemory extends FlowiseMemory implements MemoryMethods {
     appDataSource: DataSource
     databaseEntities: IDatabaseEntity
     chatflowid: string
+    orgId: string
 
     constructor(fields: BufferMemoryInput & BufferMemoryExtendedInput) {
         super(fields)
         this.appDataSource = fields.appDataSource
         this.databaseEntities = fields.databaseEntities
         this.chatflowid = fields.chatflowid
+        this.orgId = fields.orgId
     }
 
     async getChatMessages(
@@ -443,7 +448,7 @@ class BufferMemory extends FlowiseMemory implements MemoryMethods {
         }
 
         if (returnBaseMessages) {
-            return await mapChatMessageToBaseMessage(chatMessage)
+            return await mapChatMessageToBaseMessage(chatMessage, this.orgId)
         }
 
         let returnIMessages: IMessage[] = []

packages/components/nodes/chains/GraphCypherQAChain/GraphCypherQAChain.ts

@@ -215,7 +215,7 @@ class GraphCypherQA_Chain implements INode {
             query: input
         }
 
-        const loggerHandler = new ConsoleCallbackHandler(options.logger)
+        const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
         const callbackHandlers = await additionalCallbacks(nodeData, options)
         let callbacks = [loggerHandler, ...callbackHandlers]
 

packages/components/nodes/chains/LLMChain/LLMChain.ts

@@ -167,7 +167,7 @@ const runPrediction = async (
     nodeData: INodeData,
     disableStreaming?: boolean
 ) => {
-    const loggerHandler = new ConsoleCallbackHandler(options.logger)
+    const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
     const callbacks = await additionalCallbacks(nodeData, options)
 
     const moderations = nodeData.inputs?.inputModeration as Moderation[]

packages/components/nodes/chains/MultiPromptChain/MultiPromptChain.ts

@@ -66,7 +66,7 @@ class MultiPromptChain_Chains implements INode {
             promptNames,
             promptDescriptions,
             promptTemplates,
-            llmChainOpts: { verbose: process.env.DEBUG === 'true' }
+            llmChainOpts: { verbose: process.env.DEBUG === 'true' ? true : false }
         })
 
         return chain
@@ -95,7 +95,7 @@ class MultiPromptChain_Chains implements INode {
         }
         const obj = { input }
 
-        const loggerHandler = new ConsoleCallbackHandler(options.logger)
+        const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
         const callbacks = await additionalCallbacks(nodeData, options)
 
         if (shouldStreamResponse) {

packages/components/nodes/chains/MultiRetrievalQAChain/MultiRetrievalQAChain.ts

@@ -74,7 +74,7 @@ class MultiRetrievalQAChain_Chains implements INode {
             retrieverNames,
             retrieverDescriptions,
             retrievers,
-            retrievalQAChainOpts: { verbose: process.env.DEBUG === 'true', returnSourceDocuments }
+            retrievalQAChainOpts: { verbose: process.env.DEBUG === 'true' ? true : false, returnSourceDocuments }
         })
         return chain
     }
@@ -101,7 +101,7 @@ class MultiRetrievalQAChain_Chains implements INode {
             }
         }
         const obj = { input }
-        const loggerHandler = new ConsoleCallbackHandler(options.logger)
+        const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
         const callbacks = await additionalCallbacks(nodeData, options)
 
         if (shouldStreamResponse) {

packages/components/nodes/chains/RetrievalQAChain/RetrievalQAChain.ts

@@ -53,7 +53,7 @@ class RetrievalQAChain_Chains implements INode {
         const model = nodeData.inputs?.model as BaseLanguageModel
         const vectorStoreRetriever = nodeData.inputs?.vectorStoreRetriever as BaseRetriever
 
-        const chain = RetrievalQAChain.fromLLM(model, vectorStoreRetriever, { verbose: process.env.DEBUG === 'true' })
+        const chain = RetrievalQAChain.fromLLM(model, vectorStoreRetriever, { verbose: process.env.DEBUG === 'true' ? true : false })
         return chain
     }
 
@@ -80,7 +80,7 @@ class RetrievalQAChain_Chains implements INode {
         const obj = {
             query: input
         }
-        const loggerHandler = new ConsoleCallbackHandler(options.logger)
+        const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
         const callbacks = await additionalCallbacks(nodeData, options)
 
         if (shouldStreamResponse) {

packages/components/nodes/chains/SqlDatabaseChain/SqlDatabaseChain.ts

@@ -194,7 +194,7 @@ class SqlDatabaseChain_Chains implements INode {
             topK,
             customPrompt
         )
-        const loggerHandler = new ConsoleCallbackHandler(options.logger)
+        const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
         const callbacks = await additionalCallbacks(nodeData, options)
 
         if (shouldStreamResponse) {
@@ -241,7 +241,7 @@ const getSQLDBChain = async (
     const obj: SqlDatabaseChainInput = {
         llm,
         database: db,
-        verbose: process.env.DEBUG === 'true',
+        verbose: process.env.DEBUG === 'true' ? true : false,
         topK: topK
     }
 

packages/components/nodes/chains/VectorDBQAChain/VectorDBQAChain.ts

@@ -55,7 +55,7 @@ class VectorDBQAChain_Chains implements INode {
 
         const chain = VectorDBQAChain.fromLLM(model, vectorStore, {
             k: (vectorStore as any)?.k ?? 4,
-            verbose: process.env.DEBUG === 'true'
+            verbose: process.env.DEBUG === 'true' ? true : false
         })
         return chain
     }
@@ -84,7 +84,7 @@ class VectorDBQAChain_Chains implements INode {
             query: input
         }
 
-        const loggerHandler = new ConsoleCallbackHandler(options.logger)
+        const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
         const callbacks = await additionalCallbacks(nodeData, options)
 
         if (shouldStreamResponse) {

packages/components/nodes/chatmodels/AzureChatOpenAI/README.md

@@ -4,13 +4,13 @@ Azure OpenAI Chat Model integration for Flowise
 
 ## 🌱 Env Variables
 
-| Variable                     | Description                                                                                     | Type                                             | Default                             |
-| ---------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- |
-| AZURE_OPENAI_API_KEY    | Default `credential.azureOpenAIApiKey` for Azure OpenAI Model                                            | String                                                                    |            |
-| AZURE_OPENAI_API_INSTANCE_NAME    | Default `credential.azureOpenAIApiInstanceName` for Azure OpenAI Model                                            | String                                                                    |            |
-| AZURE_OPENAI_API_DEPLOYMENT_NAME    | Default `credential.azureOpenAIApiDeploymentName` for Azure OpenAI Model                                            | String                                                                    |            |
-| AZURE_OPENAI_API_VERSION    | Default `credential.azureOpenAIApiVersion` for Azure OpenAI Model                                            | String                                                                    |            |
+| Variable                         | Description                                                              | Type   | Default |
+| -------------------------------- | ------------------------------------------------------------------------ | ------ | ------- |
+| AZURE_OPENAI_API_KEY             | Default `credential.azureOpenAIApiKey` for Azure OpenAI Model            | String |         |
+| AZURE_OPENAI_API_INSTANCE_NAME   | Default `credential.azureOpenAIApiInstanceName` for Azure OpenAI Model   | String |         |
+| AZURE_OPENAI_API_DEPLOYMENT_NAME | Default `credential.azureOpenAIApiDeploymentName` for Azure OpenAI Model | String |         |
+| AZURE_OPENAI_API_VERSION         | Default `credential.azureOpenAIApiVersion` for Azure OpenAI Model        | String |         |
 
 ## License
 
-Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).
+Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).

packages/components/nodes/chatmodels/ChatGoogleGenerativeAI/ChatGoogleGenerativeAI.ts

@@ -216,6 +216,10 @@ class GoogleGenerativeAI_ChatModels implements INode {
             streaming: streaming ?? true
         }
 
+        // this extra metadata is needed, as langchain does not show the model name in the callbacks.
+        obj.metadata = {
+            fw_model_name: customModelName || modelName
+        }
         if (maxOutputTokens) obj.maxOutputTokens = parseInt(maxOutputTokens, 10)
         if (topP) obj.topP = parseFloat(topP)
         if (topK) obj.topK = parseFloat(topK)

packages/components/nodes/chatmodels/ChatIBMWatsonx/ChatIBMWatsonx.ts

@@ -161,12 +161,13 @@ class ChatIBMWatsonx_ChatModels implements INode {
             watsonxAIBearerToken
         }
 
-        const obj: ChatWatsonxInput & WatsonxAuth = {
+        const obj = {
             ...auth,
             streaming: streaming ?? true,
             model: modelName,
             temperature: temperature ? parseFloat(temperature) : undefined
-        }
+        } as ChatWatsonxInput & WatsonxAuth
+
         if (cache) obj.cache = cache
         if (maxTokens) obj.maxTokens = parseInt(maxTokens, 10)
         if (frequencyPenalty) obj.frequencyPenalty = parseInt(frequencyPenalty, 10)

packages/components/nodes/documentloaders/Cheerio/Cheerio.ts

@@ -123,6 +123,7 @@ class Cheerio_DocumentLoaders implements INode {
         const selectedLinks = nodeData.inputs?.selectedLinks as string[]
         let limit = parseInt(nodeData.inputs?.limit as string)
         const output = nodeData.outputs?.output as string
+        const orgId = options.orgId
 
         const _omitMetadataKeys = nodeData.inputs?.omitMetadataKeys as string
 
@@ -149,7 +150,8 @@ class Cheerio_DocumentLoaders implements INode {
             try {
                 let docs: IDocument[] = []
                 if (url.endsWith('.pdf')) {
-                    if (process.env.DEBUG === 'true') options.logger.info(`CheerioWebBaseLoader does not support PDF files: ${url}`)
+                    if (process.env.DEBUG === 'true')
+                        options.logger.info(`[${orgId}]: CheerioWebBaseLoader does not support PDF files: ${url}`)
                     return docs
                 }
                 const loader = new CheerioWebBaseLoader(url, params)
@@ -161,7 +163,8 @@ class Cheerio_DocumentLoaders implements INode {
                 }
                 return docs
             } catch (err) {
-                if (process.env.DEBUG === 'true') options.logger.error(`error in CheerioWebBaseLoader: ${err.message}, on page: ${url}`)
+                if (process.env.DEBUG === 'true')
+                    options.logger.error(`[${orgId}]: Error in CheerioWebBaseLoader: ${err.message}, on page: ${url}`)
                 return []
             }
         }
@@ -169,7 +172,7 @@ class Cheerio_DocumentLoaders implements INode {
         let docs: IDocument[] = []
 
         if (relativeLinksMethod) {
-            if (process.env.DEBUG === 'true') options.logger.info(`Start ${relativeLinksMethod}`)
+            if (process.env.DEBUG === 'true') options.logger.info(`[${orgId}]: Start CheerioWebBaseLoader ${relativeLinksMethod}`)
             // if limit is 0 we don't want it to default to 10 so we check explicitly for null or undefined
             // so when limit is 0 we can fetch all the links
             if (limit === null || limit === undefined) limit = 10
@@ -180,15 +183,18 @@ class Cheerio_DocumentLoaders implements INode {
                     : relativeLinksMethod === 'webCrawl'
                     ? await webCrawl(url, limit)
                     : await xmlScrape(url, limit)
-            if (process.env.DEBUG === 'true') options.logger.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
+            if (process.env.DEBUG === 'true')
+                options.logger.info(`[${orgId}]: CheerioWebBaseLoader pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
             if (!pages || pages.length === 0) throw new Error('No relative links found')
             for (const page of pages) {
                 docs.push(...(await cheerioLoader(page)))
             }
-            if (process.env.DEBUG === 'true') options.logger.info(`Finish ${relativeLinksMethod}`)
+            if (process.env.DEBUG === 'true') options.logger.info(`[${orgId}]: Finish CheerioWebBaseLoader ${relativeLinksMethod}`)
         } else if (selectedLinks && selectedLinks.length > 0) {
             if (process.env.DEBUG === 'true')
-                options.logger.info(`pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`)
+                options.logger.info(
+                    `[${orgId}]: CheerioWebBaseLoader pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`
+                )
             for (const page of selectedLinks.slice(0, limit)) {
                 docs.push(...(await cheerioLoader(page)))
             }

packages/components/nodes/documentloaders/Csv/Csv.ts

@@ -107,9 +107,9 @@ class Csv_DocumentLoaders implements INode {
         return { files, fromStorage }
     }
 
-    async getFileData(file: string, { chatflowid }: { chatflowid: string }, fromStorage?: boolean) {
+    async getFileData(file: string, { orgId, chatflowid }: { orgId: string; chatflowid: string }, fromStorage?: boolean) {
         if (fromStorage) {
-            return getFileFromStorage(file, chatflowid)
+            return getFileFromStorage(file, orgId, chatflowid)
         } else {
             const splitDataURI = file.split(',')
             splitDataURI.pop()
@@ -126,6 +126,7 @@ class Csv_DocumentLoaders implements INode {
 
         let docs: IDocument[] = []
 
+        const orgId = options.orgId
         const chatflowid = options.chatflowid
 
         const { files, fromStorage } = this.getFiles(nodeData)
@@ -133,7 +134,7 @@ class Csv_DocumentLoaders implements INode {
         for (const file of files) {
             if (!file) continue
 
-            const fileData = await this.getFileData(file, { chatflowid }, fromStorage)
+            const fileData = await this.getFileData(file, { orgId, chatflowid }, fromStorage)
             const blob = new Blob([fileData])
             const loader = new CSVLoader(blob, columnName.trim().length === 0 ? undefined : columnName.trim())
 

packages/components/nodes/documentloaders/CustomDocumentLoader/CustomDocumentLoader.ts

@@ -72,7 +72,7 @@ class CustomDocumentLoader_DocumentLoaders implements INode {
         const appDataSource = options.appDataSource as DataSource
         const databaseEntities = options.databaseEntities as IDatabaseEntity
 
-        const variables = await getVars(appDataSource, databaseEntities, nodeData)
+        const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
         const flow = {
             chatflowId: options.chatflowid,
             sessionId: options.sessionId,

packages/components/nodes/documentloaders/DocumentStore/DocStoreLoader.ts

@@ -60,7 +60,8 @@ class DocStore_DocumentLoaders implements INode {
                 return returnData
             }
 
-            const stores = await appDataSource.getRepository(databaseEntities['DocumentStore']).find()
+            const searchOptions = options.searchOptions || {}
+            const stores = await appDataSource.getRepository(databaseEntities['DocumentStore']).findBy(searchOptions)
             for (const store of stores) {
                 if (store.status === 'SYNC') {
                     const obj = {

packages/components/nodes/documentloaders/Docx/Docx.ts

@@ -96,11 +96,12 @@ class Docx_DocumentLoaders implements INode {
             } else {
                 files = [fileName]
             }
+            const orgId = options.orgId
             const chatflowid = options.chatflowid
 
             for (const file of files) {
                 if (!file) continue
-                const fileData = await getFileFromStorage(file, chatflowid)
+                const fileData = await getFileFromStorage(file, orgId, chatflowid)
                 const blob = new Blob([fileData])
                 const loader = new DocxLoader(blob)
 

packages/components/nodes/documentloaders/Epub/Epub.ts

@@ -118,10 +118,11 @@ class Epub_DocumentLoaders implements INode {
                 files = fileName.startsWith('[') && fileName.endsWith(']') ? JSON.parse(fileName) : [fileName]
 
                 const chatflowid = options.chatflowid
+                const orgId = options.orgId
 
                 for (const file of files) {
                     if (!file) continue
-                    const fileData = await getFileFromStorage(file, chatflowid)
+                    const fileData = await getFileFromStorage(file, orgId, chatflowid)
                     const tempFilePath = path.join(tempDir, `${Date.now()}_${file}`)
                     fs.writeFileSync(tempFilePath, fileData)
                     await this.extractDocs(usage, tempFilePath, textSplitter, docs)

packages/components/nodes/documentloaders/File/File.ts

@@ -144,6 +144,7 @@ class File_DocumentLoaders implements INode {
             } else {
                 files = [fileName]
             }
+            const orgId = options.orgId
             const chatflowid = options.chatflowid
 
             // specific to createAttachment to get files from chatId
@@ -151,14 +152,14 @@ class File_DocumentLoaders implements INode {
             if (retrieveAttachmentChatId) {
                 for (const file of files) {
                     if (!file) continue
-                    const fileData = await getFileFromStorage(file, chatflowid, options.chatId)
+                    const fileData = await getFileFromStorage(file, orgId, chatflowid, options.chatId)
                     const blob = new Blob([fileData])
                     fileBlobs.push({ blob, ext: file.split('.').pop() || '' })
                 }
             } else {
                 for (const file of files) {
                     if (!file) continue
-                    const fileData = await getFileFromStorage(file, chatflowid)
+                    const fileData = await getFileFromStorage(file, orgId, chatflowid)
                     const blob = new Blob([fileData])
                     fileBlobs.push({ blob, ext: file.split('.').pop() || '' })
                 }

packages/components/nodes/documentloaders/Json/Json.ts

@@ -146,11 +146,12 @@ class Json_DocumentLoaders implements INode {
             } else {
                 files = [fileName]
             }
+            const orgId = options.orgId
             const chatflowid = options.chatflowid
 
             for (const file of files) {
                 if (!file) continue
-                const fileData = await getFileFromStorage(file, chatflowid)
+                const fileData = await getFileFromStorage(file, orgId, chatflowid)
                 const blob = new Blob([fileData])
                 const loader = new JSONLoader(blob, pointers.length != 0 ? pointers : undefined, metadata)
 

packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts

@@ -135,11 +135,12 @@ class Jsonlines_DocumentLoaders implements INode {
             } else {
                 files = [fileName]
             }
+            const orgId = options.orgId
             const chatflowid = options.chatflowid
 
             for (const file of files) {
                 if (!file) continue
-                const fileData = await getFileFromStorage(file, chatflowid)
+                const fileData = await getFileFromStorage(file, orgId, chatflowid)
                 const blob = new Blob([fileData])
                 const loader = new JSONLinesLoader(blob, pointer, metadata)
 

packages/components/nodes/documentloaders/Pdf/Pdf.ts

@@ -122,11 +122,12 @@ class Pdf_DocumentLoaders implements INode {
             } else {
                 files = [fileName]
             }
+            const orgId = options.orgId
             const chatflowid = options.chatflowid
 
             for (const file of files) {
                 if (!file) continue
-                const fileData = await getFileFromStorage(file, chatflowid)
+                const fileData = await getFileFromStorage(file, orgId, chatflowid)
                 const bf = Buffer.from(fileData)
                 await this.extractDocs(usage, bf, legacyBuild, textSplitter, docs)
             }

packages/components/nodes/documentloaders/Playwright/Playwright.ts

@@ -159,6 +159,7 @@ class Playwright_DocumentLoaders implements INode {
         let waitForSelector = nodeData.inputs?.waitForSelector as string
         const _omitMetadataKeys = nodeData.inputs?.omitMetadataKeys as string
         const output = nodeData.outputs?.output as string
+        const orgId = options.orgId
 
         let omitMetadataKeys: string[] = []
         if (_omitMetadataKeys) {
@@ -202,13 +203,14 @@ class Playwright_DocumentLoaders implements INode {
                 }
                 return docs
             } catch (err) {
-                if (process.env.DEBUG === 'true') options.logger.error(`error in PlaywrightWebBaseLoader: ${err.message}, on page: ${url}`)
+                if (process.env.DEBUG === 'true')
+                    options.logger.error(`[${orgId}]: Error in PlaywrightWebBaseLoader: ${err.message}, on page: ${url}`)
             }
         }
 
         let docs: IDocument[] = []
         if (relativeLinksMethod) {
-            if (process.env.DEBUG === 'true') options.logger.info(`Start ${relativeLinksMethod}`)
+            if (process.env.DEBUG === 'true') options.logger.info(`[${orgId}]: Start PlaywrightWebBaseLoader ${relativeLinksMethod}`)
             // if limit is 0 we don't want it to default to 10 so we check explicitly for null or undefined
             // so when limit is 0 we can fetch all the links
             if (limit === null || limit === undefined) limit = 10
@@ -219,15 +221,18 @@ class Playwright_DocumentLoaders implements INode {
                     : relativeLinksMethod === 'webCrawl'
                     ? await webCrawl(url, limit)
                     : await xmlScrape(url, limit)
-            if (process.env.DEBUG === 'true') options.logger.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
+            if (process.env.DEBUG === 'true')
+                options.logger.info(`[${orgId}]: PlaywrightWebBaseLoader pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
             if (!pages || pages.length === 0) throw new Error('No relative links found')
             for (const page of pages) {
                 docs.push(...(await playwrightLoader(page)))
             }
-            if (process.env.DEBUG === 'true') options.logger.info(`Finish ${relativeLinksMethod}`)
+            if (process.env.DEBUG === 'true') options.logger.info(`[${orgId}]: Finish PlaywrightWebBaseLoader ${relativeLinksMethod}`)
         } else if (selectedLinks && selectedLinks.length > 0) {
             if (process.env.DEBUG === 'true')
-                options.logger.info(`pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`)
+                options.logger.info(
+                    `[${orgId}]: PlaywrightWebBaseLoader pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`
+                )
             for (const page of selectedLinks.slice(0, limit)) {
                 docs.push(...(await playwrightLoader(page)))
             }

packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts

@@ -155,6 +155,7 @@ class Puppeteer_DocumentLoaders implements INode {
         let waitForSelector = nodeData.inputs?.waitForSelector as string
         const _omitMetadataKeys = nodeData.inputs?.omitMetadataKeys as string
         const output = nodeData.outputs?.output as string
+        const orgId = options.orgId
 
         let omitMetadataKeys: string[] = []
         if (_omitMetadataKeys) {
@@ -198,13 +199,14 @@ class Puppeteer_DocumentLoaders implements INode {
                 }
                 return docs
             } catch (err) {
-                if (process.env.DEBUG === 'true') options.logger.error(`error in PuppeteerWebBaseLoader: ${err.message}, on page: ${url}`)
+                if (process.env.DEBUG === 'true')
+                    options.logger.error(`[${orgId}]: Error in PuppeteerWebBaseLoader: ${err.message}, on page: ${url}`)
             }
         }
 
         let docs: IDocument[] = []
         if (relativeLinksMethod) {
-            if (process.env.DEBUG === 'true') options.logger.info(`Start ${relativeLinksMethod}`)
+            if (process.env.DEBUG === 'true') options.logger.info(`[${orgId}]: Start PuppeteerWebBaseLoader ${relativeLinksMethod}`)
             // if limit is 0 we don't want it to default to 10 so we check explicitly for null or undefined
             // so when limit is 0 we can fetch all the links
             if (limit === null || limit === undefined) limit = 10
@@ -215,15 +217,18 @@ class Puppeteer_DocumentLoaders implements INode {
                     : relativeLinksMethod === 'webCrawl'
                     ? await webCrawl(url, limit)
                     : await xmlScrape(url, limit)
-            if (process.env.DEBUG === 'true') options.logger.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
+            if (process.env.DEBUG === 'true')
+                options.logger.info(`[${orgId}]: PuppeteerWebBaseLoader pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
             if (!pages || pages.length === 0) throw new Error('No relative links found')
             for (const page of pages) {
                 docs.push(...(await puppeteerLoader(page)))
             }
-            if (process.env.DEBUG === 'true') options.logger.info(`Finish ${relativeLinksMethod}`)
+            if (process.env.DEBUG === 'true') options.logger.info(`[${orgId}]: Finish PuppeteerWebBaseLoader ${relativeLinksMethod}`)
         } else if (selectedLinks && selectedLinks.length > 0) {
             if (process.env.DEBUG === 'true')
-                options.logger.info(`pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`)
+                options.logger.info(
+                    `[${orgId}]: PuppeteerWebBaseLoader pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`
+                )
             for (const page of selectedLinks.slice(0, limit)) {
                 docs.push(...(await puppeteerLoader(page)))
             }

packages/components/nodes/documentloaders/S3File/README.md

@@ -4,10 +4,10 @@ DS File Loarder integration for Flowise
 
 ## 🌱 Env Variables
 
-| Variable                     | Description                                                                                     | Type                                             | Default                             |
-| ---------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- |
-| UNSTRUCTURED_API_URL | Default `unstructuredApiUrl` for S3 File Loader                                            | String                                                                    |  http://localhost:8000/general/v0/general          |
+| Variable             | Description                                     | Type   | Default                                  |
+| -------------------- | ----------------------------------------------- | ------ | ---------------------------------------- |
+| UNSTRUCTURED_API_URL | Default `unstructuredApiUrl` for S3 File Loader | String | http://localhost:8000/general/v0/general |
 
 ## License
 
-Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).
+Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).

packages/components/nodes/documentloaders/Text/Text.ts

@@ -98,11 +98,12 @@ class Text_DocumentLoaders implements INode {
             } else {
                 files = [fileName]
             }
+            const orgId = options.orgId
             const chatflowid = options.chatflowid
 
             for (const file of files) {
                 if (!file) continue
-                const fileData = await getFileFromStorage(file, chatflowid)
+                const fileData = await getFileFromStorage(file, orgId, chatflowid)
                 const blob = new Blob([fileData])
                 const loader = new TextLoader(blob)
 

packages/components/nodes/documentloaders/Unstructured/README.md

@@ -4,10 +4,10 @@ Unstructured File Loader integration for Flowise
 
 ## 🌱 Env Variables
 
-| Variable                     | Description                                                                                     | Type                                             | Default                             |
-| ---------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- |
-| UNSTRUCTURED_API_URL | Default `apiUrl` for Unstructured File/Floder Loader                                            | String                                                                    |  http://localhost:8000/general/v0/general          |
+| Variable             | Description                                          | Type   | Default                                  |
+| -------------------- | ---------------------------------------------------- | ------ | ---------------------------------------- |
+| UNSTRUCTURED_API_URL | Default `apiUrl` for Unstructured File/Floder Loader | String | http://localhost:8000/general/v0/general |
 
 ## License
 
-Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).
+Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).

packages/components/nodes/documentloaders/Unstructured/UnstructuredFile.ts

@@ -532,11 +532,12 @@ class UnstructuredFile_DocumentLoaders implements INode {
                 } else {
                     files = [fileName]
                 }
+                const orgId = options.orgId
                 const chatflowid = options.chatflowid
 
                 for (const file of files) {
                     if (!file) continue
-                    const fileData = await getFileFromStorage(file, chatflowid)
+                    const fileData = await getFileFromStorage(file, orgId, chatflowid)
                     const loaderDocs = await loader.loadAndSplitBuffer(fileData, file)
                     docs.push(...loaderDocs)
                 }

packages/components/nodes/embeddings/AzureOpenAIEmbedding/README.md

@@ -4,13 +4,13 @@ Azure OpenAI Embedding Model integration for Flowise
 
 ## 🌱 Env Variables
 
-| Variable                     | Description                                                                                     | Type                                             | Default                             |
-| ---------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- |
-| AZURE_OPENAI_API_KEY    | Default `credential.azureOpenAIApiKey` for Azure OpenAI Model                                            | String                                                                    |            |
-| AZURE_OPENAI_API_INSTANCE_NAME    | Default `credential.azureOpenAIApiInstanceName` for Azure OpenAI Model                                            | String                                                                    |            |
-| AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME    | Default `credential.azureOpenAIApiDeploymentName` for Azure OpenAI Model                                            | String                                                                    |            |
-| AZURE_OPENAI_API_VERSION    | Default `credential.azureOpenAIApiVersion` for Azure OpenAI Model                                            | String                                                                    |            |
+| Variable                                    | Description                                                              | Type   | Default |
+| ------------------------------------------- | ------------------------------------------------------------------------ | ------ | ------- |
+| AZURE_OPENAI_API_KEY                        | Default `credential.azureOpenAIApiKey` for Azure OpenAI Model            | String |         |
+| AZURE_OPENAI_API_INSTANCE_NAME              | Default `credential.azureOpenAIApiInstanceName` for Azure OpenAI Model   | String |         |
+| AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME | Default `credential.azureOpenAIApiDeploymentName` for Azure OpenAI Model | String |         |
+| AZURE_OPENAI_API_VERSION                    | Default `credential.azureOpenAIApiVersion` for Azure OpenAI Model        | String |         |
 
 ## License
 
-Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).
+Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).

packages/components/nodes/engine/ChatEngine/ContextChatEngine.ts

@@ -10,6 +10,7 @@ import {
 } from '../../../src/Interface'
 import { Metadata, BaseRetriever, LLM, ContextChatEngine, ChatMessage, NodeWithScore } from 'llamaindex'
 import { reformatSourceDocuments } from '../EngineUtils'
+import { EvaluationRunTracerLlama } from '../../../evaluation/EvaluationRunTracerLlama'
 
 class ContextChatEngine_LlamaIndex implements INode {
     label: string
@@ -93,6 +94,9 @@ class ContextChatEngine_LlamaIndex implements INode {
 
         const chatEngine = new ContextChatEngine({ chatModel: model, retriever: vectorStoreRetriever })
 
+        // these are needed for evaluation runs
+        await EvaluationRunTracerLlama.injectEvaluationMetadata(nodeData, options, chatEngine)
+
         const msgs = (await memory.getChatMessages(this.sessionId, false, prependMessages)) as IMessage[]
         for (const message of msgs) {
             if (message.type === 'apiMessage') {

packages/components/nodes/engine/ChatEngine/SimpleChatEngine.ts

@@ -9,6 +9,7 @@ import {
     IServerSideEventStreamer
 } from '../../../src/Interface'
 import { LLM, ChatMessage, SimpleChatEngine } from 'llamaindex'
+import { EvaluationRunTracerLlama } from '../../../evaluation/EvaluationRunTracerLlama'
 
 class SimpleChatEngine_LlamaIndex implements INode {
     label: string
@@ -78,6 +79,9 @@ class SimpleChatEngine_LlamaIndex implements INode {
 
         const chatEngine = new SimpleChatEngine({ llm: model })
 
+        // these are needed for evaluation runs
+        await EvaluationRunTracerLlama.injectEvaluationMetadata(nodeData, options, chatEngine)
+
         const msgs = (await memory.getChatMessages(this.sessionId, false, prependMessages)) as IMessage[]
         for (const message of msgs) {
             if (message.type === 'apiMessage') {

packages/components/nodes/engine/QueryEngine/QueryEngine.ts

@@ -10,6 +10,7 @@ import {
     NodeWithScore
 } from 'llamaindex'
 import { reformatSourceDocuments } from '../EngineUtils'
+import { EvaluationRunTracerLlama } from '../../../evaluation/EvaluationRunTracerLlama'
 
 class QueryEngine_LlamaIndex implements INode {
     label: string
@@ -72,6 +73,8 @@ class QueryEngine_LlamaIndex implements INode {
         let sourceNodes: NodeWithScore<Metadata>[] = []
         let isStreamingStarted = false
 
+        await EvaluationRunTracerLlama.injectEvaluationMetadata(nodeData, options, queryEngine)
+
         const shouldStreamResponse = options.shouldStreamResponse
         const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
         const chatId = options.chatId

packages/components/nodes/engine/SubQuestionQueryEngine/SubQuestionQueryEngine.ts

@@ -15,6 +15,7 @@ import {
     NodeWithScore
 } from 'llamaindex'
 import { reformatSourceDocuments } from '../EngineUtils'
+import { EvaluationRunTracerLlama } from '../../../evaluation/EvaluationRunTracerLlama'
 
 class SubQuestionQueryEngine_LlamaIndex implements INode {
     label: string
@@ -89,6 +90,8 @@ class SubQuestionQueryEngine_LlamaIndex implements INode {
         let sourceNodes: NodeWithScore<Metadata>[] = []
         let isStreamingStarted = false
 
+        await EvaluationRunTracerLlama.injectEvaluationMetadata(nodeData, options, queryEngine)
+
         const shouldStreamResponse = options.shouldStreamResponse
         const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
         const chatId = options.chatId

packages/components/nodes/llms/Azure OpenAI/README.md

@@ -4,13 +4,13 @@ Azure OpenAI LLM integration for Flowise
 
 ## 🌱 Env Variables
 
-| Variable                     | Description                                                                                     | Type                                             | Default                             |
-| ---------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- |
-| AZURE_OPENAI_API_KEY    | Default `credential.azureOpenAIApiKey` for Azure OpenAI LLM                                            | String                                                                    |            |
-| AZURE_OPENAI_API_INSTANCE_NAME    | Default `credential.azureOpenAIApiInstanceName` for Azure OpenAI LLM                                            | String                                                                    |            |
-| AZURE_OPENAI_API_DEPLOYMENT_NAME    | Default `credential.azureOpenAIApiDeploymentName` for Azure OpenAI LLM                                            | String                                                                    |            |
-| AZURE_OPENAI_API_VERSION    | Default `credential.azureOpenAIApiVersion` for Azure OpenAI LLM                                            | String                                                                    |            |
+| Variable                         | Description                                                            | Type   | Default |
+| -------------------------------- | ---------------------------------------------------------------------- | ------ | ------- |
+| AZURE_OPENAI_API_KEY             | Default `credential.azureOpenAIApiKey` for Azure OpenAI LLM            | String |         |
+| AZURE_OPENAI_API_INSTANCE_NAME   | Default `credential.azureOpenAIApiInstanceName` for Azure OpenAI LLM   | String |         |
+| AZURE_OPENAI_API_DEPLOYMENT_NAME | Default `credential.azureOpenAIApiDeploymentName` for Azure OpenAI LLM | String |         |
+| AZURE_OPENAI_API_VERSION         | Default `credential.azureOpenAIApiVersion` for Azure OpenAI LLM        | String |         |
 
 ## License
 
-Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).
+Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).

packages/components/nodes/memory/AgentMemory/AgentMemory.ts

@@ -108,6 +108,7 @@ class AgentMemory_Memory implements INode {
         const databaseType = nodeData.inputs?.databaseType as string
         const databaseEntities = options.databaseEntities as IDatabaseEntity
         const chatflowid = options.chatflowid as string
+        const orgId = options.orgId as string
         const appDataSource = options.appDataSource as DataSource
 
         let additionalConfiguration = {}
@@ -135,7 +136,8 @@ class AgentMemory_Memory implements INode {
                 threadId,
                 appDataSource,
                 databaseEntities,
-                chatflowid
+                chatflowid,
+                orgId
             }
             const recordManager = new SqliteSaver(args)
             return recordManager
@@ -159,7 +161,8 @@ class AgentMemory_Memory implements INode {
                 threadId,
                 appDataSource,
                 databaseEntities,
-                chatflowid
+                chatflowid,
+                orgId
             }
             const recordManager = new PostgresSaver(args)
             return recordManager
@@ -184,7 +187,8 @@ class AgentMemory_Memory implements INode {
                 threadId,
                 appDataSource,
                 databaseEntities,
-                chatflowid
+                chatflowid,
+                orgId
             }
             const recordManager = new MySQLSaver(args)
             return recordManager

packages/components/nodes/memory/AgentMemory/MySQLAgentMemory/MySQLAgentMemory.ts

@@ -65,6 +65,7 @@ class MySQLAgentMemory_Memory implements INode {
         const databaseEntities = options.databaseEntities as IDatabaseEntity
         const chatflowid = options.chatflowid as string
         const appDataSource = options.appDataSource as DataSource
+        const orgId = options.orgId as string
 
         let additionalConfiguration = {}
         if (additionalConfig) {
@@ -102,7 +103,8 @@ class MySQLAgentMemory_Memory implements INode {
             threadId,
             appDataSource,
             databaseEntities,
-            chatflowid
+            chatflowid,
+            orgId
         }
         const recordManager = new MySQLSaver(args)
         return recordManager

packages/components/nodes/memory/AgentMemory/MySQLAgentMemory/mysqlSaver.ts

@@ -242,7 +242,7 @@ export class MySQLSaver extends BaseCheckpointSaver implements MemoryMethods {
         }
 
         if (returnBaseMessages) {
-            return await mapChatMessageToBaseMessage(chatMessage)
+            return await mapChatMessageToBaseMessage(chatMessage, this.config.orgId)
         }
 
         let returnIMessages: IMessage[] = []

packages/components/nodes/memory/AgentMemory/PostgresAgentMemory/PostgresAgentMemory.ts

@@ -65,6 +65,7 @@ class PostgresAgentMemory_Memory implements INode {
         const databaseEntities = options.databaseEntities as IDatabaseEntity
         const chatflowid = options.chatflowid as string
         const appDataSource = options.appDataSource as DataSource
+        const orgId = options.orgId as string
 
         let additionalConfiguration = {}
         if (additionalConfig) {
@@ -101,7 +102,8 @@ class PostgresAgentMemory_Memory implements INode {
             threadId,
             appDataSource,
             databaseEntities,
-            chatflowid
+            chatflowid,
+            orgId
         }
         const recordManager = new PostgresSaver(args)
         return recordManager

packages/components/nodes/memory/AgentMemory/PostgresAgentMemory/pgSaver.ts

@@ -283,7 +283,7 @@ CREATE TABLE IF NOT EXISTS ${tableName} (
         }
 
         if (returnBaseMessages) {
-            return await mapChatMessageToBaseMessage(chatMessage)
+            return await mapChatMessageToBaseMessage(chatMessage, this.config.orgId)
         }
 
         let returnIMessages: IMessage[] = []

packages/components/nodes/memory/AgentMemory/SQLiteAgentMemory/SQLiteAgentMemory.ts

@@ -51,6 +51,7 @@ class SQLiteAgentMemory_Memory implements INode {
         const databaseEntities = options.databaseEntities as IDatabaseEntity
         const chatflowid = options.chatflowid as string
         const appDataSource = options.appDataSource as DataSource
+        const orgId = options.orgId as string
 
         let additionalConfiguration = {}
         if (additionalConfig) {
@@ -76,7 +77,8 @@ class SQLiteAgentMemory_Memory implements INode {
             threadId,
             appDataSource,
             databaseEntities,
-            chatflowid
+            chatflowid,
+            orgId
         }
 
         const recordManager = new SqliteSaver(args)

packages/components/nodes/memory/AgentMemory/SQLiteAgentMemory/sqliteSaver.ts

@@ -266,7 +266,7 @@ CREATE TABLE IF NOT EXISTS ${tableName} (
         }
 
         if (returnBaseMessages) {
-            return await mapChatMessageToBaseMessage(chatMessage)
+            return await mapChatMessageToBaseMessage(chatMessage, this.config.orgId)
         }
 
         let returnIMessages: IMessage[] = []

packages/components/nodes/memory/AgentMemory/interface.ts

@@ -9,6 +9,7 @@ export type SaverOptions = {
     appDataSource: DataSource
     databaseEntities: IDatabaseEntity
     chatflowid: string
+    orgId: string
 }
 
 export interface CheckpointTuple {

packages/components/nodes/memory/BufferMemory/BufferMemory.ts

@@ -61,6 +61,7 @@ class BufferMemory_Memory implements INode {
         const appDataSource = options.appDataSource as DataSource
         const databaseEntities = options.databaseEntities as IDatabaseEntity
         const chatflowid = options.chatflowid as string
+        const orgId = options.orgId as string
 
         return new BufferMemoryExtended({
             returnMessages: true,
@@ -68,7 +69,8 @@ class BufferMemory_Memory implements INode {
             sessionId,
             appDataSource,
             databaseEntities,
-            chatflowid
+            chatflowid,
+            orgId
         })
     }
 }
@@ -78,12 +80,14 @@ interface BufferMemoryExtendedInput {
     appDataSource: DataSource
     databaseEntities: IDatabaseEntity
     chatflowid: string
+    orgId: string
 }
 
 class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
     appDataSource: DataSource
     databaseEntities: IDatabaseEntity
     chatflowid: string
+    orgId: string
     sessionId = ''
 
     constructor(fields: BufferMemoryInput & BufferMemoryExtendedInput) {
@@ -92,6 +96,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
         this.appDataSource = fields.appDataSource
         this.databaseEntities = fields.databaseEntities
         this.chatflowid = fields.chatflowid
+        this.orgId = fields.orgId
     }
 
     async getChatMessages(
@@ -117,7 +122,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
         }
 
         if (returnBaseMessages) {
-            return await mapChatMessageToBaseMessage(chatMessage)
+            return await mapChatMessageToBaseMessage(chatMessage, this.orgId)
         }
 
         let returnIMessages: IMessage[] = []

packages/components/nodes/memory/BufferWindowMemory/BufferWindowMemory.ts

@@ -69,6 +69,7 @@ class BufferWindowMemory_Memory implements INode {
         const appDataSource = options.appDataSource as DataSource
         const databaseEntities = options.databaseEntities as IDatabaseEntity
         const chatflowid = options.chatflowid as string
+        const orgId = options.orgId as string
 
         const obj: Partial<BufferWindowMemoryInput> & BufferMemoryExtendedInput = {
             returnMessages: true,
@@ -77,7 +78,8 @@ class BufferWindowMemory_Memory implements INode {
             k: parseInt(k, 10),
             appDataSource,
             databaseEntities,
-            chatflowid
+            chatflowid,
+            orgId
         }
 
         return new BufferWindowMemoryExtended(obj)
@@ -89,12 +91,14 @@ interface BufferMemoryExtendedInput {
     appDataSource: DataSource
     databaseEntities: IDatabaseEntity
     chatflowid: string
+    orgId: string
 }
 
 class BufferWindowMemoryExtended extends FlowiseWindowMemory implements MemoryMethods {
     appDataSource: DataSource
     databaseEntities: IDatabaseEntity
     chatflowid: string
+    orgId: string
     sessionId = ''
 
     constructor(fields: BufferWindowMemoryInput & BufferMemoryExtendedInput) {
@@ -103,6 +107,7 @@ class BufferWindowMemoryExtended extends FlowiseWindowMemory implements MemoryMe
         this.appDataSource = fields.appDataSource
         this.databaseEntities = fields.databaseEntities
         this.chatflowid = fields.chatflowid
+        this.orgId = fields.orgId
     }
 
     async getChatMessages(
@@ -134,7 +139,7 @@ class BufferWindowMemoryExtended extends FlowiseWindowMemory implements MemoryMe
         }
 
         if (returnBaseMessages) {
-            return await mapChatMessageToBaseMessage(chatMessage)
+            return await mapChatMessageToBaseMessage(chatMessage, this.orgId)
         }
 
         let returnIMessages: IMessage[] = []

packages/components/nodes/memory/ConversationSummaryBufferMemory/ConversationSummaryBufferMemory.ts

@@ -78,6 +78,7 @@ class ConversationSummaryBufferMemory_Memory implements INode {
         const appDataSource = options.appDataSource as DataSource
         const databaseEntities = options.databaseEntities as IDatabaseEntity
         const chatflowid = options.chatflowid as string
+        const orgId = options.orgId as string
 
         const obj: ConversationSummaryBufferMemoryInput & BufferMemoryExtendedInput = {
             llm: model,
@@ -87,7 +88,8 @@ class ConversationSummaryBufferMemory_Memory implements INode {
             returnMessages: true,
             appDataSource,
             databaseEntities,
-            chatflowid
+            chatflowid,
+            orgId
         }
 
         return new ConversationSummaryBufferMemoryExtended(obj)
@@ -99,12 +101,14 @@ interface BufferMemoryExtendedInput {
     appDataSource: DataSource
     databaseEntities: IDatabaseEntity
     chatflowid: string
+    orgId: string
 }
 
 class ConversationSummaryBufferMemoryExtended extends FlowiseSummaryBufferMemory implements MemoryMethods {
     appDataSource: DataSource
     databaseEntities: IDatabaseEntity
     chatflowid: string
+    orgId: string
     sessionId = ''
 
     constructor(fields: ConversationSummaryBufferMemoryInput & BufferMemoryExtendedInput) {
@@ -113,6 +117,7 @@ class ConversationSummaryBufferMemoryExtended extends FlowiseSummaryBufferMemory
         this.appDataSource = fields.appDataSource
         this.databaseEntities = fields.databaseEntities
         this.chatflowid = fields.chatflowid
+        this.orgId = fields.orgId
     }
 
     async getChatMessages(
@@ -137,7 +142,7 @@ class ConversationSummaryBufferMemoryExtended extends FlowiseSummaryBufferMemory
             chatMessage.unshift(...prependMessages)
         }
 
-        let baseMessages = await mapChatMessageToBaseMessage(chatMessage)
+        let baseMessages = await mapChatMessageToBaseMessage(chatMessage, this.orgId)
 
         // Prune baseMessages if it exceeds max token limit
         if (this.movingSummaryBuffer) {

packages/components/nodes/memory/ConversationSummaryMemory/ConversationSummaryMemory.ts

@@ -69,6 +69,7 @@ class ConversationSummaryMemory_Memory implements INode {
         const appDataSource = options.appDataSource as DataSource
         const databaseEntities = options.databaseEntities as IDatabaseEntity
         const chatflowid = options.chatflowid as string
+        const orgId = options.orgId as string
 
         const obj: ConversationSummaryMemoryInput & BufferMemoryExtendedInput = {
             llm: model,
@@ -77,7 +78,8 @@ class ConversationSummaryMemory_Memory implements INode {
             sessionId,
             appDataSource,
             databaseEntities,
-            chatflowid
+            chatflowid,
+            orgId
         }
 
         return new ConversationSummaryMemoryExtended(obj)
@@ -89,12 +91,14 @@ interface BufferMemoryExtendedInput {
     appDataSource: DataSource
     databaseEntities: IDatabaseEntity
     chatflowid: string
+    orgId: string
 }
 
 class ConversationSummaryMemoryExtended extends FlowiseSummaryMemory implements MemoryMethods {
     appDataSource: DataSource
     databaseEntities: IDatabaseEntity
     chatflowid: string
+    orgId: string
     sessionId = ''
 
     constructor(fields: ConversationSummaryMemoryInput & BufferMemoryExtendedInput) {
@@ -103,6 +107,7 @@ class ConversationSummaryMemoryExtended extends FlowiseSummaryMemory implements
         this.appDataSource = fields.appDataSource
         this.databaseEntities = fields.databaseEntities
         this.chatflowid = fields.chatflowid
+        this.orgId = fields.orgId
     }
 
     async getChatMessages(
@@ -128,7 +133,7 @@ class ConversationSummaryMemoryExtended extends FlowiseSummaryMemory implements
             chatMessage.unshift(...prependMessages)
         }
 
-        const baseMessages = await mapChatMessageToBaseMessage(chatMessage)
+        const baseMessages = await mapChatMessageToBaseMessage(chatMessage, this.orgId)
 
         // Get summary
         if (this.llm && typeof this.llm !== 'string') {

packages/components/nodes/memory/DynamoDb/DynamoDb.ts

@@ -125,6 +125,8 @@ const initializeDynamoDB = async (nodeData: INodeData, options: ICommonObject):
         config
     })
 
+    const orgId = options.orgId as string
+
     const memory = new BufferMemoryExtended({
         memoryKey: memoryKey ?? 'chat_history',
         chatHistory: dynamoDb,
@@ -132,7 +134,8 @@ const initializeDynamoDB = async (nodeData: INodeData, options: ICommonObject):
         dynamodbClient: client,
         tableName,
         partitionKey,
-        dynamoKey: { [partitionKey]: { S: sessionId } }
+        dynamoKey: { [partitionKey]: { S: sessionId } },
+        orgId
     })
     return memory
 }
@@ -143,6 +146,7 @@ interface BufferMemoryExtendedInput {
     tableName: string
     partitionKey: string
     dynamoKey: Record<string, AttributeValue>
+    orgId: string
 }
 
 interface DynamoDBSerializedChatMessage {
@@ -165,6 +169,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
     private dynamoKey: Record<string, AttributeValue>
     private messageAttributeName: string
     sessionId = ''
+    orgId = ''
     dynamodbClient: DynamoDBClient
 
     constructor(fields: BufferMemoryInput & BufferMemoryExtendedInput) {
@@ -174,6 +179,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
         this.tableName = fields.tableName
         this.partitionKey = fields.partitionKey
         this.dynamoKey = fields.dynamoKey
+        this.orgId = fields.orgId
     }
 
     overrideDynamoKey(overrideSessionId = '') {
@@ -260,7 +266,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
             .filter((x): x is StoredMessage => x.type !== undefined && x.data.content !== undefined)
         const baseMessages = messages.map(mapStoredMessageToChatMessage)
         if (prependMessages?.length) {
-            baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages)))
+            baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages, this.orgId)))
         }
         return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages)
     }

packages/components/nodes/memory/Mem0/Mem0.ts

@@ -151,6 +151,7 @@ class Mem0_Memory implements INode {
 const initializeMem0 = async (nodeData: INodeData, options: ICommonObject): Promise<BaseMem0Memory> => {
     const initialUserId = nodeData.inputs?.user_id as string
     const useFlowiseChatId = nodeData.inputs?.useFlowiseChatId as boolean
+    const orgId = options.orgId as string
 
     if (!useFlowiseChatId && !initialUserId) {
         throw new Error('User ID field cannot be empty when "Use Flowise Chat ID" is OFF.')
@@ -198,7 +199,8 @@ const initializeMem0 = async (nodeData: INodeData, options: ICommonObject): Prom
             databaseEntities: options.databaseEntities as IDatabaseEntity,
             chatflowid: options.chatflowid as string,
             searchOnly: (nodeData.inputs?.searchOnly as boolean) || false,
-            useFlowiseChatId: useFlowiseChatId
+            useFlowiseChatId: useFlowiseChatId,
+            orgId: orgId
         }
 
     return new Mem0MemoryExtended(obj)
@@ -207,11 +209,13 @@ const initializeMem0 = async (nodeData: INodeData, options: ICommonObject): Prom
 interface Mem0MemoryExtendedInput extends Mem0MemoryInput {
     memoryOptions?: MemoryOptions | SearchOptions
     useFlowiseChatId: boolean
+    orgId: string
 }
 
 class Mem0MemoryExtended extends BaseMem0Memory implements MemoryMethods {
     initialUserId: string
     userId: string
+    orgId: string
     memoryKey: string
     inputKey: string
     appDataSource: DataSource
@@ -233,6 +237,7 @@ class Mem0MemoryExtended extends BaseMem0Memory implements MemoryMethods {
         this.chatflowid = fields.chatflowid
         this.searchOnly = fields.searchOnly
         this.useFlowiseChatId = fields.useFlowiseChatId
+        this.orgId = fields.orgId
     }
 
     // Selects Mem0 user_id based on toggle state (Flowise chat ID or input field)
@@ -337,7 +342,7 @@ class Mem0MemoryExtended extends BaseMem0Memory implements MemoryMethods {
                 console.warn('Mem0 history is not a string, cannot prepend directly.')
             }
 
-            return await mapChatMessageToBaseMessage(chatMessage)
+            return await mapChatMessageToBaseMessage(chatMessage, this.orgId)
         }
 
         return returnIMessages

packages/components/nodes/memory/MongoDBMemory/MongoDBMemory.ts

@@ -88,9 +88,12 @@ const initializeMongoDB = async (nodeData: INodeData, options: ICommonObject): P
     const mongoDBConnectUrl = getCredentialParam('mongoDBConnectUrl', credentialData, nodeData)
     const driverInfo = { name: 'Flowise', version: (await getVersion()).version }
 
+    const orgId = options.orgId as string
+
     return new BufferMemoryExtended({
         memoryKey: memoryKey ?? 'chat_history',
         sessionId,
+        orgId,
         mongoConnection: {
             databaseName,
             collectionName,
@@ -102,6 +105,7 @@ const initializeMongoDB = async (nodeData: INodeData, options: ICommonObject): P
 
 interface BufferMemoryExtendedInput {
     sessionId: string
+    orgId: string
     mongoConnection: {
         databaseName: string
         collectionName: string
@@ -112,6 +116,7 @@ interface BufferMemoryExtendedInput {
 
 class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
     sessionId = ''
+    orgId = ''
     mongoConnection: {
         databaseName: string
         collectionName: string
@@ -122,6 +127,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
     constructor(fields: BufferMemoryInput & BufferMemoryExtendedInput) {
         super(fields)
         this.sessionId = fields.sessionId
+        this.orgId = fields.orgId
         this.mongoConnection = fields.mongoConnection
     }
 
@@ -138,7 +144,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
         const messages = document?.messages || []
         const baseMessages = messages.map(mapStoredMessageToChatMessage)
         if (prependMessages?.length) {
-            baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages)))
+            baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages, this.orgId)))
         }
 
         await client.close()

packages/components/nodes/memory/RedisBackedChatMemory/RedisBackedChatMemory.ts

@@ -88,6 +88,7 @@ const initializeRedis = async (nodeData: INodeData, options: ICommonObject): Pro
 
     const credentialData = await getCredentialData(nodeData.credential ?? '', options)
     const redisUrl = getCredentialParam('redisUrl', credentialData, nodeData)
+    const orgId = options.orgId as string
 
     const redisOptions = redisUrl
         ? redisUrl
@@ -104,7 +105,8 @@ const initializeRedis = async (nodeData: INodeData, options: ICommonObject): Pro
         sessionId,
         windowSize,
         sessionTTL,
-        redisOptions
+        redisOptions,
+        orgId
     })
 
     return memory
@@ -114,11 +116,13 @@ interface BufferMemoryExtendedInput {
     sessionId: string
     windowSize?: number
     sessionTTL?: number
+    orgId: string
     redisOptions: RedisOptions | string
 }
 
 class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
     sessionId = ''
+    orgId = ''
     windowSize?: number
     sessionTTL?: number
     redisOptions: RedisOptions | string
@@ -128,6 +132,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
         this.sessionId = fields.sessionId
         this.windowSize = fields.windowSize
         this.sessionTTL = fields.sessionTTL
+        this.orgId = fields.orgId
         this.redisOptions = fields.redisOptions
     }
 
@@ -165,7 +170,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
             const orderedMessages = rawStoredMessages.reverse().map((message) => JSON.parse(message))
             const baseMessages = orderedMessages.map(mapStoredMessageToChatMessage)
             if (prependMessages?.length) {
-                baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages)))
+                baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages, this.orgId)))
             }
             return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages)
         })

packages/components/nodes/memory/UpstashRedisBackedChatMemory/UpstashRedisBackedChatMemory.ts

@@ -100,13 +100,14 @@ const initalizeUpstashRedis = async (nodeData: INodeData, options: ICommonObject
         sessionTTL,
         client
     })
-
+    const orgId = options.orgId as string
     const memory = new BufferMemoryExtended({
         memoryKey: memoryKey ?? 'chat_history',
         chatHistory: redisChatMessageHistory,
         sessionId,
         sessionTTL,
-        redisClient: client
+        redisClient: client,
+        orgId
     })
 
     return memory
@@ -115,11 +116,13 @@ const initalizeUpstashRedis = async (nodeData: INodeData, options: ICommonObject
 interface BufferMemoryExtendedInput {
     redisClient: Redis
     sessionId: string
+    orgId: string
     sessionTTL?: number
 }
 
 class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
     sessionId = ''
+    orgId = ''
     redisClient: Redis
     sessionTTL?: number
 
@@ -128,6 +131,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
         this.sessionId = fields.sessionId
         this.redisClient = fields.redisClient
         this.sessionTTL = fields.sessionTTL
+        this.orgId = fields.orgId
     }
 
     async getChatMessages(
@@ -143,7 +147,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
         const previousMessages = orderedMessages.filter((x): x is StoredMessage => x.type !== undefined && x.data.content !== undefined)
         const baseMessages = previousMessages.map(mapStoredMessageToChatMessage)
         if (prependMessages?.length) {
-            baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages)))
+            baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages, this.orgId)))
         }
         return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages)
     }

packages/components/nodes/memory/ZepMemory/ZepMemory.ts

@@ -119,6 +119,7 @@ const initializeZep = async (nodeData: INodeData, options: ICommonObject): Promi
     const credentialData = await getCredentialData(nodeData.credential ?? '', options)
     const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
 
+    const orgId = options.orgId as string
     const obj: ZepMemoryInput & ZepMemoryExtendedInput = {
         baseURL,
         aiPrefix,
@@ -127,6 +128,7 @@ const initializeZep = async (nodeData: INodeData, options: ICommonObject): Promi
         memoryKey,
         inputKey,
         sessionId,
+        orgId,
         k: k ? parseInt(k, 10) : undefined
     }
     if (apiKey) obj.apiKey = apiKey
@@ -136,14 +138,17 @@ const initializeZep = async (nodeData: INodeData, options: ICommonObject): Promi
 
 interface ZepMemoryExtendedInput {
     k?: number
+    orgId: string
 }
 
 class ZepMemoryExtended extends ZepMemory implements MemoryMethods {
     lastN?: number
+    orgId = ''
 
     constructor(fields: ZepMemoryInput & ZepMemoryExtendedInput) {
         super(fields)
         this.lastN = fields.k
+        this.orgId = fields.orgId
     }
 
     async loadMemoryVariables(values: InputValues, overrideSessionId = ''): Promise<MemoryVariables> {
@@ -176,7 +181,7 @@ class ZepMemoryExtended extends ZepMemory implements MemoryMethods {
         const memoryVariables = await this.loadMemoryVariables({}, id)
         const baseMessages = memoryVariables[this.memoryKey]
         if (prependMessages?.length) {
-            baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages)))
+            baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages, this.orgId)))
         }
         return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages)
     }

packages/components/nodes/memory/ZepMemoryCloud/ZepMemoryCloud.ts

@@ -113,6 +113,7 @@ const initializeZep = async (nodeData: INodeData, options: ICommonObject): Promi
 
     const credentialData = await getCredentialData(nodeData.credential ?? '', options)
     const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
+    const orgId = options.orgId as string
     const obj: ZepMemoryInput & ZepMemoryExtendedInput = {
         apiKey,
         aiPrefix,
@@ -121,7 +122,8 @@ const initializeZep = async (nodeData: INodeData, options: ICommonObject): Promi
         sessionId,
         inputKey,
         memoryType: memoryType,
-        returnMessages: true
+        returnMessages: true,
+        orgId
     }
 
     return new ZepMemoryExtended(obj)
@@ -129,14 +131,17 @@ const initializeZep = async (nodeData: INodeData, options: ICommonObject): Promi
 
 interface ZepMemoryExtendedInput {
     memoryType?: 'perpetual' | 'message_window'
+    orgId: string
 }
 
 class ZepMemoryExtended extends ZepMemory implements MemoryMethods {
     memoryType: 'perpetual' | 'message_window'
+    orgId: string
 
     constructor(fields: ZepMemoryInput & ZepMemoryExtendedInput) {
         super(fields)
         this.memoryType = fields.memoryType ?? 'perpetual'
+        this.orgId = fields.orgId
     }
 
     async loadMemoryVariables(values: InputValues, overrideSessionId = ''): Promise<MemoryVariables> {
@@ -169,7 +174,7 @@ class ZepMemoryExtended extends ZepMemory implements MemoryMethods {
         const memoryVariables = await this.loadMemoryVariables({}, id)
         const baseMessages = memoryVariables[this.memoryKey]
         if (prependMessages?.length) {
-            baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages)))
+            baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages, this.orgId)))
         }
         return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages)
     }

packages/components/nodes/multiagents/Worker/Worker.ts

@@ -233,7 +233,7 @@ async function createAgent(
             sessionId: flowObj?.sessionId,
             chatId: flowObj?.chatId,
             input: flowObj?.input,
-            verbose: process.env.DEBUG === 'true',
+            verbose: process.env.DEBUG === 'true' ? true : false,
             maxIterations: maxIterations ? parseFloat(maxIterations) : undefined
         })
         return executor

packages/components/nodes/prompts/ChatPromptTemplate/ChatPromptTemplate.ts

@@ -120,7 +120,7 @@ class ChatPromptTemplate_Prompts implements INode {
         ) {
             const appDataSource = options.appDataSource as DataSource
             const databaseEntities = options.databaseEntities as IDatabaseEntity
-            const vm = await getVM(appDataSource, databaseEntities, nodeData, {})
+            const vm = await getVM(appDataSource, databaseEntities, nodeData, options, {})
             try {
                 const response = await vm.run(`module.exports = async function() {${messageHistoryCode}}()`, __dirname)
                 if (!Array.isArray(response)) throw new Error('Returned message history must be an array')

packages/components/nodes/sequentialagents/Agent/Agent.ts

@@ -680,7 +680,7 @@ async function createAgent(
             sessionId: flowObj?.sessionId,
             chatId: flowObj?.chatId,
             input: flowObj?.input,
-            verbose: process.env.DEBUG === 'true',
+            verbose: process.env.DEBUG === 'true' ? true : false,
             maxIterations: maxIterations ? parseFloat(maxIterations) : undefined
         })
         return executor
@@ -877,7 +877,7 @@ const getReturnOutput = async (nodeData: INodeData, input: string, options: ICom
     const updateStateMemory = nodeData.inputs?.updateStateMemory as string
 
     const selectedTab = tabIdentifier ? tabIdentifier.split(`_${nodeData.id}`)[0] : 'updateStateMemoryUI'
-    const variables = await getVars(appDataSource, databaseEntities, nodeData)
+    const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
 
     const flow = {
         chatflowId: options.chatflowid,
@@ -930,7 +930,7 @@ const getReturnOutput = async (nodeData: INodeData, input: string, options: ICom
             throw new Error(e)
         }
     } else if (selectedTab === 'updateStateMemoryCode' && updateStateMemoryCode) {
-        const vm = await getVM(appDataSource, databaseEntities, nodeData, flow)
+        const vm = await getVM(appDataSource, databaseEntities, nodeData, options, flow)
         try {
             const response = await vm.run(`module.exports = async function() {${updateStateMemoryCode}}()`, __dirname)
             if (typeof response !== 'object') throw new Error('Return output must be an object')

packages/components/nodes/sequentialagents/Condition/Condition.ts

@@ -267,7 +267,7 @@ const runCondition = async (nodeData: INodeData, input: string, options: ICommon
     const tabIdentifier = nodeData.inputs?.[`${TAB_IDENTIFIER}_${nodeData.id}`] as string
 
     const selectedTab = tabIdentifier ? tabIdentifier.split(`_${nodeData.id}`)[0] : 'conditionUI'
-    const variables = await getVars(appDataSource, databaseEntities, nodeData)
+    const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
 
     const flow = {
         chatflowId: options.chatflowid,
@@ -279,7 +279,7 @@ const runCondition = async (nodeData: INodeData, input: string, options: ICommon
     }
 
     if (selectedTab === 'conditionFunction' && conditionFunction) {
-        const vm = await getVM(appDataSource, databaseEntities, nodeData, flow)
+        const vm = await getVM(appDataSource, databaseEntities, nodeData, options, flow)
         try {
             const response = await vm.run(`module.exports = async function() {${conditionFunction}}()`, __dirname)
             if (typeof response !== 'string') throw new Error('Condition function must return a string')

packages/components/nodes/sequentialagents/ConditionAgent/ConditionAgent.ts

@@ -540,7 +540,7 @@ const runCondition = async (
         result = { ...jsonResult, additional_kwargs: { nodeId: nodeData.id } }
     }
 
-    const variables = await getVars(appDataSource, databaseEntities, nodeData)
+    const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
 
     const flow = {
         chatflowId: options.chatflowid,
@@ -553,7 +553,7 @@ const runCondition = async (
     }
 
     if (selectedTab === 'conditionFunction' && conditionFunction) {
-        const vm = await getVM(appDataSource, databaseEntities, nodeData, flow)
+        const vm = await getVM(appDataSource, databaseEntities, nodeData, options, flow)
         try {
             const response = await vm.run(`module.exports = async function() {${conditionFunction}}()`, __dirname)
             if (typeof response !== 'string') throw new Error('Condition function must return a string')

packages/components/nodes/sequentialagents/CustomFunction/CustomFunction.ts

@@ -102,7 +102,7 @@ class CustomFunction_SeqAgents implements INode {
         if (!sequentialNodes || !sequentialNodes.length) throw new Error('Custom function must have a predecessor!')
 
         const executeFunc = async (state: ISeqAgentsState) => {
-            const variables = await getVars(appDataSource, databaseEntities, nodeData)
+            const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
             const flow = {
                 chatflowId: options.chatflowid,
                 sessionId: options.sessionId,

packages/components/nodes/sequentialagents/ExecuteFlow/ExecuteFlow.ts

@@ -141,7 +141,8 @@ class ExecuteFlow_SeqAgents implements INode {
                 return returnData
             }
 
-            const chatflows = await appDataSource.getRepository(databaseEntities['ChatFlow']).find()
+            const searchOptions = options.searchOptions || {}
+            const chatflows = await appDataSource.getRepository(databaseEntities['ChatFlow']).findBy(searchOptions)
 
             for (let i = 0; i < chatflows.length; i += 1) {
                 const data = {
@@ -189,7 +190,7 @@ class ExecuteFlow_SeqAgents implements INode {
         const chatId = options.chatId
 
         const executeFunc = async (state: ISeqAgentsState) => {
-            const variables = await getVars(appDataSource, databaseEntities, nodeData)
+            const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
 
             let flowInput = ''
             if (seqExecuteFlowInput === 'userQuestion') {
@@ -223,7 +224,7 @@ class ExecuteFlow_SeqAgents implements INode {
                 }
             }
 
-            const options = {
+            const callOptions = {
                 method: 'POST',
                 headers: {
                     'Content-Type': 'application/json',
@@ -234,7 +235,7 @@ class ExecuteFlow_SeqAgents implements INode {
 
             let sandbox: ICommonObject = {
                 $input: flowInput,
-                $callOptions: options,
+                $callOptions: callOptions,
                 $callBody: body,
                 util: undefined,
                 Symbol: undefined,

packages/components/nodes/sequentialagents/LLMNode/LLMNode.ts

@@ -668,7 +668,7 @@ const getReturnOutput = async (nodeData: INodeData, input: string, options: ICom
     const updateStateMemory = nodeData.inputs?.updateStateMemory as string
 
     const selectedTab = tabIdentifier ? tabIdentifier.split(`_${nodeData.id}`)[0] : 'updateStateMemoryUI'
-    const variables = await getVars(appDataSource, databaseEntities, nodeData)
+    const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
 
     const flow = {
         chatflowId: options.chatflowid,
@@ -721,7 +721,7 @@ const getReturnOutput = async (nodeData: INodeData, input: string, options: ICom
             throw new Error(e)
         }
     } else if (selectedTab === 'updateStateMemoryCode' && updateStateMemoryCode) {
-        const vm = await getVM(appDataSource, databaseEntities, nodeData, flow)
+        const vm = await getVM(appDataSource, databaseEntities, nodeData, options, flow)
         try {
             const response = await vm.run(`module.exports = async function() {${updateStateMemoryCode}}()`, __dirname)
             if (typeof response !== 'object') throw new Error('Return output must be an object')

packages/components/nodes/sequentialagents/State/State.ts

@@ -190,7 +190,7 @@ class State_SeqAgents implements INode {
                 throw new Error(e)
             }
         } else if (selectedTab === 'stateMemoryCode' && stateMemoryCode) {
-            const variables = await getVars(appDataSource, databaseEntities, nodeData)
+            const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
             const flow = {
                 chatflowId: options.chatflowid,
                 sessionId: options.sessionId,

packages/components/nodes/sequentialagents/ToolNode/ToolNode.ts

@@ -498,7 +498,7 @@ const getReturnOutput = async (
     const updateStateMemory = nodeData.inputs?.updateStateMemory as string
 
     const selectedTab = tabIdentifier ? tabIdentifier.split(`_${nodeData.id}`)[0] : 'updateStateMemoryUI'
-    const variables = await getVars(appDataSource, databaseEntities, nodeData)
+    const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
 
     const reformattedOutput = outputs.map((output) => {
         return {
@@ -561,7 +561,7 @@ const getReturnOutput = async (
             throw new Error(e)
         }
     } else if (selectedTab === 'updateStateMemoryCode' && updateStateMemoryCode) {
-        const vm = await getVM(appDataSource, databaseEntities, nodeData, flow)
+        const vm = await getVM(appDataSource, databaseEntities, nodeData, options, flow)
         try {
             const response = await vm.run(`module.exports = async function() {${updateStateMemoryCode}}()`, __dirname)
             if (typeof response !== 'object') throw new Error('Return output must be an object')

packages/components/nodes/sequentialagents/commonUtils.ts

@@ -150,8 +150,14 @@ export const processImageMessage = async (llm: BaseChatModel, nodeData: INodeDat
     return multiModalMessageContent
 }
 
-export const getVM = async (appDataSource: DataSource, databaseEntities: IDatabaseEntity, nodeData: INodeData, flow: ICommonObject) => {
-    const variables = await getVars(appDataSource, databaseEntities, nodeData)
+export const getVM = async (
+    appDataSource: DataSource,
+    databaseEntities: IDatabaseEntity,
+    nodeData: INodeData,
+    options: ICommonObject,
+    flow: ICommonObject
+) => {
+    const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
 
     let sandbox: any = {
         util: undefined,
@@ -420,7 +426,7 @@ export const checkMessageHistory = async (
     if (messageHistory) {
         const appDataSource = options.appDataSource as DataSource
         const databaseEntities = options.databaseEntities as IDatabaseEntity
-        const vm = await getVM(appDataSource, databaseEntities, nodeData, {})
+        const vm = await getVM(appDataSource, databaseEntities, nodeData, options, {})
         try {
             const response = await vm.run(`module.exports = async function() {${messageHistory}}()`, __dirname)
             if (!Array.isArray(response)) throw new Error('Returned message history must be an array')

packages/components/nodes/tools/ChainTool/core.ts

@@ -14,17 +14,41 @@ export class ChainTool extends DynamicTool {
         super({
             ...rest,
             func: async (input, runManager) => {
-                const childManagers = runManager?.getChild()
-                const handlers = childManagers?.handlers?.filter((handler) => !(handler instanceof CustomChainHandler)) || []
-                if (childManagers) childManagers.handlers = handlers
+                // prevent sending SSE events of the sub-chain
+                const sseStreamer = runManager?.handlers.find((handler) => handler instanceof CustomChainHandler)?.sseStreamer
+                if (runManager) {
+                    const callbacks = runManager.handlers
+                    for (let i = 0; i < callbacks.length; i += 1) {
+                        if (callbacks[i] instanceof CustomChainHandler) {
+                            ;(callbacks[i] as any).sseStreamer = undefined
+                        }
+                    }
+                }
 
                 if ((chain as any).prompt && (chain as any).prompt.promptValues) {
                     const promptValues = handleEscapeCharacters((chain as any).prompt.promptValues, true)
-                    const values = await chain.call(promptValues, childManagers)
+
+                    const values = await chain.call(promptValues, runManager?.getChild())
+                    if (runManager && sseStreamer) {
+                        const callbacks = runManager.handlers
+                        for (let i = 0; i < callbacks.length; i += 1) {
+                            if (callbacks[i] instanceof CustomChainHandler) {
+                                ;(callbacks[i] as any).sseStreamer = sseStreamer
+                            }
+                        }
+                    }
                     return values?.text
                 }
 
-                const values = chain.run(input, childManagers)
+                const values = chain.run(input, runManager?.getChild())
+                if (runManager && sseStreamer) {
+                    const callbacks = runManager.handlers
+                    for (let i = 0; i < callbacks.length; i += 1) {
+                        if (callbacks[i] instanceof CustomChainHandler) {
+                            ;(callbacks[i] as any).sseStreamer = sseStreamer
+                        }
+                    }
+                }
                 return values
             }
         })

packages/components/nodes/tools/ChatflowTool/ChatflowTool.ts

@@ -122,7 +122,8 @@ class ChatflowTool_Tools implements INode {
                 return returnData
             }
 
-            const chatflows = await appDataSource.getRepository(databaseEntities['ChatFlow']).find()
+            const searchOptions = options.searchOptions || {}
+            const chatflows = await appDataSource.getRepository(databaseEntities['ChatFlow']).findBy(searchOptions)
 
             for (let i = 0; i < chatflows.length; i += 1) {
                 const data = {

packages/components/nodes/tools/CodeInterpreterE2B/CodeInterpreterE2B.ts

@@ -80,7 +80,8 @@ class Code_Interpreter_Tools implements INode {
             schema: z.object({
                 input: z.string().describe('Python code to be executed in the sandbox environment')
             }),
-            chatflowid: options.chatflowid
+            chatflowid: options.chatflowid,
+            orgId: options.orgId
         })
     }
 }
@@ -92,6 +93,7 @@ type E2BToolInput = {
     apiKey: string
     schema: any
     chatflowid: string
+    orgId: string
     templateCodeInterpreterE2B?: string
     domainCodeInterpreterE2B?: string
 }
@@ -113,6 +115,8 @@ export class E2BTool extends StructuredTool {
 
     chatflowid: string
 
+    orgId: string
+
     flowObj: ICommonObject
 
     templateCodeInterpreterE2B?: string
@@ -125,6 +129,7 @@ export class E2BTool extends StructuredTool {
         this.apiKey = options.apiKey
         this.schema = options.schema
         this.chatflowid = options.chatflowid
+        this.orgId = options.orgId
         this.templateCodeInterpreterE2B = options.templateCodeInterpreterE2B
         this.domainCodeInterpreterE2B = options.domainCodeInterpreterE2B
     }
@@ -136,6 +141,7 @@ export class E2BTool extends StructuredTool {
             apiKey: options.apiKey,
             schema: options.schema,
             chatflowid: options.chatflowid,
+            orgId: options.orgId,
             templateCodeInterpreterE2B: options.templateCodeInterpreterE2B,
             domainCodeInterpreterE2B: options.domainCodeInterpreterE2B
         })
@@ -212,28 +218,33 @@ export class E2BTool extends StructuredTool {
 
                             const filename = `artifact_${Date.now()}.png`
 
-                            const res = await addSingleFileToStorage(
+                            // Don't check storage usage because this is incoming file, and if we throw error, agent will keep on retrying
+                            const { path } = await addSingleFileToStorage(
                                 'image/png',
                                 pngData,
                                 filename,
+                                this.orgId,
                                 this.chatflowid,
                                 flowConfig!.chatId as string
                             )
-                            artifacts.push({ type: 'png', data: res })
+
+                            artifacts.push({ type: 'png', data: path })
                         } else if (key === 'jpeg') {
                             //@ts-ignore
                             const jpegData = Buffer.from(result.jpeg, 'base64')
 
                             const filename = `artifact_${Date.now()}.jpg`
 
-                            const res = await addSingleFileToStorage(
+                            const { path } = await addSingleFileToStorage(
                                 'image/jpg',
                                 jpegData,
                                 filename,
+                                this.orgId,
                                 this.chatflowid,
                                 flowConfig!.chatId as string
                             )
-                            artifacts.push({ type: 'jpeg', data: res })
+
+                            artifacts.push({ type: 'jpeg', data: path })
                         } else if (key === 'html' || key === 'markdown' || key === 'latex' || key === 'json' || key === 'javascript') {
                             artifacts.push({ type: key, data: (result as any)[key] })
                         } //TODO: support for pdf

packages/components/nodes/tools/CustomTool/CustomTool.ts

@@ -77,7 +77,8 @@ class CustomTool_Tools implements INode {
                 return returnData
             }
 
-            const tools = await appDataSource.getRepository(databaseEntities['Tool']).find()
+            const searchOptions = options.searchOptions || {}
+            const tools = await appDataSource.getRepository(databaseEntities['Tool']).findBy(searchOptions)
 
             for (let i = 0; i < tools.length; i += 1) {
                 const data = {
@@ -122,7 +123,7 @@ class CustomTool_Tools implements INode {
                 obj.schema = zodSchemaFunction(z)
             }
 
-            const variables = await getVars(appDataSource, databaseEntities, nodeData)
+            const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
 
             const flow = { chatflowId: options.chatflowid }
 

packages/components/nodes/tools/OpenAPIToolkit/OpenAPIToolkit.ts

@@ -85,8 +85,9 @@ class OpenAPIToolkit_Tools implements INode {
         let data
         if (yamlFileBase64.startsWith('FILE-STORAGE::')) {
             const file = yamlFileBase64.replace('FILE-STORAGE::', '')
+            const orgId = options.orgId
             const chatflowid = options.chatflowid
-            const fileData = await getFileFromStorage(file, chatflowid)
+            const fileData = await getFileFromStorage(file, orgId, chatflowid)
             const utf8String = fileData.toString('utf-8')
 
             data = load(utf8String)
@@ -110,7 +111,7 @@ class OpenAPIToolkit_Tools implements INode {
 
         const appDataSource = options.appDataSource as DataSource
         const databaseEntities = options.databaseEntities as IDatabaseEntity
-        const variables = await getVars(appDataSource, databaseEntities, nodeData)
+        const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
 
         const flow = { chatflowId: options.chatflowid }
 

packages/components/nodes/utilities/CustomFunction/CustomFunction.ts

@@ -83,7 +83,7 @@ class CustomFunction_Utilities implements INode {
         const databaseEntities = options.databaseEntities as IDatabaseEntity
         const tools = Object.fromEntries((flatten(nodeData.inputs?.tools) as StructuredTool[])?.map((tool) => [tool.name, tool]) ?? [])
 
-        const variables = await getVars(appDataSource, databaseEntities, nodeData)
+        const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
         const flow = {
             chatflowId: options.chatflowid,
             sessionId: options.sessionId,

packages/components/nodes/utilities/IfElseFunction/IfElseFunction.ts

@@ -85,7 +85,7 @@ class IfElseFunction_Utilities implements INode {
         const appDataSource = options.appDataSource as DataSource
         const databaseEntities = options.databaseEntities as IDatabaseEntity
 
-        const variables = await getVars(appDataSource, databaseEntities, nodeData)
+        const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
         const flow = {
             chatflowId: options.chatflowid,
             sessionId: options.sessionId,

packages/components/nodes/vectorstores/Chroma/Chroma.ts

@@ -213,7 +213,6 @@ class Chroma_VectorStores implements INode {
         const chromaApiKey = getCredentialParam('chromaApiKey', credentialData, nodeData)
         const chromaTenant = getCredentialParam('chromaTenant', credentialData, nodeData)
         const chromaDatabase = getCredentialParam('chromaDatabase', credentialData, nodeData)
-
         const chromaMetadataFilter = nodeData.inputs?.chromaMetadataFilter
 
         const obj: {

packages/components/nodes/vectorstores/DocumentStoreVS/DocStoreVector.ts

@@ -56,7 +56,8 @@ class DocStore_VectorStores implements INode {
                 return returnData
             }
 
-            const stores = await appDataSource.getRepository(databaseEntities['DocumentStore']).find()
+            const searchOptions = options.searchOptions || {}
+            const stores = await appDataSource.getRepository(databaseEntities['DocumentStore']).findBy(searchOptions)
             for (const store of stores) {
                 if (store.status === 'UPSERTED') {
                     const obj = {

packages/components/nodes/vectorstores/Vectara/Vectara.ts

@@ -191,11 +191,12 @@ class Vectara_VectorStores implements INode {
                 } else {
                     files = [fileName]
                 }
+                const orgId = options.orgId
                 const chatflowid = options.chatflowid
 
                 for (const file of files) {
                     if (!file) continue
-                    const fileData = await getFileFromStorage(file, chatflowid)
+                    const fileData = await getFileFromStorage(file, orgId, chatflowid)
                     const blob = new Blob([fileData])
                     vectaraFiles.push({ blob: blob, fileName: getFileName(file) })
                 }

packages/components/nodes/vectorstores/Vectara/Vectara_Upload.ts

@@ -0,0 +1,197 @@
+import { VectaraStore, VectaraLibArgs, VectaraFilter, VectaraContextConfig, VectaraFile } from '@langchain/community/vectorstores/vectara'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { getFileFromStorage } from '../../../src'
+
+class VectaraUpload_VectorStores implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    badge: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    credential: INodeParams
+    outputs: INodeOutputsValue[]
+
+    constructor() {
+        this.label = 'Vectara Upload File'
+        this.name = 'vectaraUpload'
+        this.version = 1.0
+        this.type = 'Vectara'
+        this.icon = 'vectara.png'
+        this.category = 'Vector Stores'
+        this.description = 'Upload files to Vectara'
+        this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
+        this.badge = 'DEPRECATING'
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['vectaraApi']
+        }
+        this.inputs = [
+            {
+                label: 'File',
+                name: 'file',
+                description:
+                    'File to upload to Vectara. Supported file types: https://docs.vectara.com/docs/api-reference/indexing-apis/file-upload/file-upload-filetypes',
+                type: 'file'
+            },
+            {
+                label: 'Metadata Filter',
+                name: 'filter',
+                description:
+                    'Filter to apply to Vectara metadata. Refer to the <a target="_blank" href="https://docs.flowiseai.com/vector-stores/vectara">documentation</a> on how to use Vectara filters with Flowise.',
+                type: 'string',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Sentences Before',
+                name: 'sentencesBefore',
+                description: 'Number of sentences to fetch before the matched sentence. Defaults to 2.',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Sentences After',
+                name: 'sentencesAfter',
+                description: 'Number of sentences to fetch after the matched sentence. Defaults to 2.',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Lambda',
+                name: 'lambda',
+                description:
+                    'Improves retrieval accuracy by adjusting the balance (from 0 to 1) between neural search and keyword-based search factors.',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Top K',
+                name: 'topK',
+                description: 'Number of top results to fetch. Defaults to 4',
+                placeholder: '4',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            }
+        ]
+        this.outputs = [
+            {
+                label: 'Vectara Retriever',
+                name: 'retriever',
+                baseClasses: this.baseClasses
+            },
+            {
+                label: 'Vectara Vector Store',
+                name: 'vectorStore',
+                baseClasses: [this.type, ...getBaseClasses(VectaraStore)]
+            }
+        ]
+    }
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
+        const customerId = getCredentialParam('customerID', credentialData, nodeData)
+        const corpusId = getCredentialParam('corpusID', credentialData, nodeData).split(',')
+
+        const fileBase64 = nodeData.inputs?.file
+        const vectaraMetadataFilter = nodeData.inputs?.filter as string
+        const sentencesBefore = nodeData.inputs?.sentencesBefore as number
+        const sentencesAfter = nodeData.inputs?.sentencesAfter as number
+        const lambda = nodeData.inputs?.lambda as number
+        const output = nodeData.outputs?.output as string
+        const topK = nodeData.inputs?.topK as string
+        const k = topK ? parseInt(topK, 10) : 4
+
+        const vectaraArgs: VectaraLibArgs = {
+            apiKey: apiKey,
+            customerId: customerId,
+            corpusId: corpusId,
+            source: 'flowise'
+        }
+
+        const vectaraFilter: VectaraFilter = {}
+        if (vectaraMetadataFilter) vectaraFilter.filter = vectaraMetadataFilter
+        if (lambda) vectaraFilter.lambda = lambda
+
+        const vectaraContextConfig: VectaraContextConfig = {}
+        if (sentencesBefore) vectaraContextConfig.sentencesBefore = sentencesBefore
+        if (sentencesAfter) vectaraContextConfig.sentencesAfter = sentencesAfter
+        vectaraFilter.contextConfig = vectaraContextConfig
+
+        let files: string[] = []
+        const vectaraFiles: VectaraFile[] = []
+
+        if (fileBase64.startsWith('FILE-STORAGE::')) {
+            const fileName = fileBase64.replace('FILE-STORAGE::', '')
+            if (fileName.startsWith('[') && fileName.endsWith(']')) {
+                files = JSON.parse(fileName)
+            } else {
+                files = [fileName]
+            }
+            const orgId = options.orgId
+            const chatflowid = options.chatflowid
+
+            for (const file of files) {
+                const fileData = await getFileFromStorage(file, orgId, chatflowid)
+                const blob = new Blob([fileData])
+                vectaraFiles.push({ blob: blob, fileName: getFileName(file) })
+            }
+        } else {
+            if (fileBase64.startsWith('[') && fileBase64.endsWith(']')) {
+                files = JSON.parse(fileBase64)
+            } else {
+                files = [fileBase64]
+            }
+
+            for (const file of files) {
+                const splitDataURI = file.split(',')
+                splitDataURI.pop()
+                const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
+                const blob = new Blob([bf])
+                vectaraFiles.push({ blob: blob, fileName: getFileName(file) })
+            }
+        }
+
+        const vectorStore = new VectaraStore(vectaraArgs)
+        await vectorStore.addFiles(vectaraFiles)
+
+        if (output === 'retriever') {
+            const retriever = vectorStore.asRetriever(k, vectaraFilter)
+            return retriever
+        } else if (output === 'vectorStore') {
+            ;(vectorStore as any).k = k
+            return vectorStore
+        }
+        return vectorStore
+    }
+}
+
+const getFileName = (fileBase64: string) => {
+    let fileNames = []
+    if (fileBase64.startsWith('[') && fileBase64.endsWith(']')) {
+        const files = JSON.parse(fileBase64)
+        for (const file of files) {
+            const splitDataURI = file.split(',')
+            const filename = splitDataURI[splitDataURI.length - 1].split(':')[1]
+            fileNames.push(filename)
+        }
+        return fileNames.join(', ')
+    } else {
+        const splitDataURI = fileBase64.split(',')
+        const filename = splitDataURI[splitDataURI.length - 1].split(':')[1]
+        return filename
+    }
+}
+
+module.exports = { nodeClass: VectaraUpload_VectorStores }

packages/components/src/Interface.Evaluation.ts

@@ -0,0 +1,43 @@
+// Evaluation Related Interfaces
+export interface IDataset {
+    id: string
+    name: string
+    createdDate: Date
+    updatedDate: Date
+}
+export interface IDatasetRow {
+    id: string
+    datasetId: string
+    input: string
+    output: string
+    updatedDate: Date
+}
+
+export enum EvaluationStatus {
+    PENDING = 'pending',
+    COMPLETED = 'completed'
+}
+export interface IEvaluation {
+    id: string
+    name: string
+    chatflowId: string
+    chatflowName: string
+    datasetId: string
+    datasetName: string
+    evaluationType: string
+    average_metrics: string
+    status: string
+    runDate: Date
+}
+
+export interface IEvaluationRun {
+    id: string
+    evaluationId: string
+    input: string
+    expectedOutput: string
+    actualOutput: string
+    metrics: string
+    runDate: Date
+    reasoning: string
+    score: number
+}

packages/components/src/Interface.ts

@@ -414,11 +414,14 @@ export interface IVisionChatModal {
     revertToOriginalModel(): void
     setMultiModalOption(multiModalOption: IMultiModalOption): void
 }
+
 export interface IStateWithMessages extends ICommonObject {
     messages: BaseMessage[]
     [key: string]: any
 }
 
+export * from './Interface.Evaluation'
+
 export interface IServerSideEventStreamer {
     streamStartEvent(chatId: string, data: any): void
     streamTokenEvent(chatId: string, data: string): void

packages/components/src/MetricsLogger.ts

@@ -0,0 +1,135 @@
+import { BaseTracer, Run } from '@langchain/core/tracers/base'
+import { Logger } from 'winston'
+import { AgentRun, elapsed, tryJsonStringify } from './handler'
+
+export class MetricsLogger extends BaseTracer {
+    name = 'console_callback_handler' as const
+    logger: Logger
+    orgId?: string
+
+    protected persistRun(_run: Run) {
+        return Promise.resolve()
+    }
+
+    constructor(logger: Logger, orgId?: string) {
+        super()
+        this.logger = logger
+        this.orgId = orgId
+    }
+
+    // utility methods
+
+    getParents(run: Run) {
+        const parents: Run[] = []
+        let currentRun = run
+        while (currentRun.parent_run_id) {
+            const parent = this.runMap.get(currentRun.parent_run_id)
+            if (parent) {
+                parents.push(parent)
+                currentRun = parent
+            } else {
+                break
+            }
+        }
+        return parents
+    }
+
+    getBreadcrumbs(run: Run) {
+        const parents = this.getParents(run).reverse()
+        const string = [...parents, run]
+            .map((parent) => {
+                const name = `${parent.execution_order}:${parent.run_type}:${parent.name}`
+                return name
+            })
+            .join(' > ')
+        return string
+    }
+
+    // logging methods
+
+    onChainStart(run: Run) {
+        const crumbs = this.getBreadcrumbs(run)
+        this.logger.verbose(
+            `[${this.orgId}]: [chain/start] [${crumbs}] Entering Chain run with input: ${tryJsonStringify(run.inputs, '[inputs]')}`
+        )
+    }
+
+    onChainEnd(run: Run) {
+        const crumbs = this.getBreadcrumbs(run)
+        this.logger.verbose(
+            `[${this.orgId}]: [chain/end] [${crumbs}] [${elapsed(run)}] Exiting Chain run with output: ${tryJsonStringify(
+                run.outputs,
+                '[outputs]'
+            )}`
+        )
+    }
+
+    onChainError(run: Run) {
+        const crumbs = this.getBreadcrumbs(run)
+        this.logger.verbose(
+            `[${this.orgId}]: [chain/error] [${crumbs}] [${elapsed(run)}] Chain run errored with error: ${tryJsonStringify(
+                run.error,
+                '[error]'
+            )}`
+        )
+    }
+
+    onLLMStart(run: Run) {
+        const crumbs = this.getBreadcrumbs(run)
+        const inputs = 'prompts' in run.inputs ? { prompts: (run.inputs.prompts as string[]).map((p) => p.trim()) } : run.inputs
+        this.logger.verbose(`[${this.orgId}]: [llm/start] [${crumbs}] Entering LLM run with input: ${tryJsonStringify(inputs, '[inputs]')}`)
+    }
+
+    onLLMEnd(run: Run) {
+        const crumbs = this.getBreadcrumbs(run)
+        this.logger.verbose(
+            `[${this.orgId}]: [llm/end] [${crumbs}] [${elapsed(run)}] Exiting LLM run with output: ${tryJsonStringify(
+                run.outputs,
+                '[response]'
+            )}`
+        )
+    }
+
+    onLLMError(run: Run) {
+        const crumbs = this.getBreadcrumbs(run)
+        this.logger.verbose(
+            `[${this.orgId}]: [llm/error] [${crumbs}] [${elapsed(run)}] LLM run errored with error: ${tryJsonStringify(
+                run.error,
+                '[error]'
+            )}`
+        )
+    }
+
+    onToolStart(run: Run) {
+        const crumbs = this.getBreadcrumbs(run)
+        this.logger.verbose(`[${this.orgId}]: [tool/start] [${crumbs}] Entering Tool run with input: "${run.inputs.input?.trim()}"`)
+    }
+
+    onToolEnd(run: Run) {
+        const crumbs = this.getBreadcrumbs(run)
+        this.logger.verbose(
+            `[${this.orgId}]: [tool/end] [${crumbs}] [${elapsed(run)}] Exiting Tool run with output: "${run.outputs?.output?.trim()}"`
+        )
+    }
+
+    onToolError(run: Run) {
+        const crumbs = this.getBreadcrumbs(run)
+        this.logger.verbose(
+            `[${this.orgId}]: [tool/error] [${crumbs}] [${elapsed(run)}] Tool run errored with error: ${tryJsonStringify(
+                run.error,
+                '[error]'
+            )}`
+        )
+    }
+
+    onAgentAction(run: Run) {
+        const agentRun = run as AgentRun
+        const crumbs = this.getBreadcrumbs(run)
+        this.logger.verbose(
+            `[${this.orgId}]: [agent/action] [${crumbs}] Agent selected action: ${tryJsonStringify(
+                agentRun.actions[agentRun.actions.length - 1],
+                '[action]'
+            )}`
+        )
+    }
+}

packages/components/src/followUpPrompts.ts

@@ -36,6 +36,7 @@ export const generateFollowUpPrompts = async (
                     model: providerConfig.modelName,
                     temperature: parseFloat(`${providerConfig.temperature}`)
                 })
+                // @ts-ignore
                 const structuredLLM = llm.withStructuredOutput(FollowUpPromptType)
                 const structuredResponse = await structuredLLM.invoke(followUpPromptsPrompt)
                 return structuredResponse