Chore/refractor (#4454)

* markdown files and env examples cleanup

* components update

* update jsonlines description

* server refractor

* update telemetry

* add execute custom node

* add ui refractor

* add username and password authenticate

* correctly retrieve past images in agentflowv2

* disable e2e temporarily

* add existing username and password authenticate

* update migration to default workspace

* update todo

* blob storage migrating

* throw error on agent tool call error

* add missing execution import

* add referral

* chore: add error message when importData is undefined

* migrate api keys to db

* fix: data too long for column executionData

* migrate api keys from json to db at init

* add info on account setup

* update docstore missing fields

---------

Co-authored-by: chungyau97 <chungyau97@gmail.com>

packages/components/nodes/sequentialagents/Agent/Agent.ts

@@ -680,7 +680,7 @@ async function createAgent(
             sessionId: flowObj?.sessionId,
             chatId: flowObj?.chatId,
             input: flowObj?.input,
-            verbose: process.env.DEBUG === 'true',
+            verbose: process.env.DEBUG === 'true' ? true : false,
             maxIterations: maxIterations ? parseFloat(maxIterations) : undefined
         })
         return executor
@@ -877,7 +877,7 @@ const getReturnOutput = async (nodeData: INodeData, input: string, options: ICom
     const updateStateMemory = nodeData.inputs?.updateStateMemory as string
 
     const selectedTab = tabIdentifier ? tabIdentifier.split(`_${nodeData.id}`)[0] : 'updateStateMemoryUI'
-    const variables = await getVars(appDataSource, databaseEntities, nodeData)
+    const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
 
     const flow = {
         chatflowId: options.chatflowid,
@@ -930,7 +930,7 @@ const getReturnOutput = async (nodeData: INodeData, input: string, options: ICom
             throw new Error(e)
         }
     } else if (selectedTab === 'updateStateMemoryCode' && updateStateMemoryCode) {
-        const vm = await getVM(appDataSource, databaseEntities, nodeData, flow)
+        const vm = await getVM(appDataSource, databaseEntities, nodeData, options, flow)
         try {
             const response = await vm.run(`module.exports = async function() {${updateStateMemoryCode}}()`, __dirname)
             if (typeof response !== 'object') throw new Error('Return output must be an object')

packages/components/nodes/sequentialagents/Condition/Condition.ts

@@ -267,7 +267,7 @@ const runCondition = async (nodeData: INodeData, input: string, options: ICommon
     const tabIdentifier = nodeData.inputs?.[`${TAB_IDENTIFIER}_${nodeData.id}`] as string
 
     const selectedTab = tabIdentifier ? tabIdentifier.split(`_${nodeData.id}`)[0] : 'conditionUI'
-    const variables = await getVars(appDataSource, databaseEntities, nodeData)
+    const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
 
     const flow = {
         chatflowId: options.chatflowid,
@@ -279,7 +279,7 @@ const runCondition = async (nodeData: INodeData, input: string, options: ICommon
     }
 
     if (selectedTab === 'conditionFunction' && conditionFunction) {
-        const vm = await getVM(appDataSource, databaseEntities, nodeData, flow)
+        const vm = await getVM(appDataSource, databaseEntities, nodeData, options, flow)
         try {
             const response = await vm.run(`module.exports = async function() {${conditionFunction}}()`, __dirname)
             if (typeof response !== 'string') throw new Error('Condition function must return a string')

packages/components/nodes/sequentialagents/ConditionAgent/ConditionAgent.ts

@@ -540,7 +540,7 @@ const runCondition = async (
         result = { ...jsonResult, additional_kwargs: { nodeId: nodeData.id } }
     }
 
-    const variables = await getVars(appDataSource, databaseEntities, nodeData)
+    const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
 
     const flow = {
         chatflowId: options.chatflowid,
@@ -553,7 +553,7 @@ const runCondition = async (
     }
 
     if (selectedTab === 'conditionFunction' && conditionFunction) {
-        const vm = await getVM(appDataSource, databaseEntities, nodeData, flow)
+        const vm = await getVM(appDataSource, databaseEntities, nodeData, options, flow)
         try {
             const response = await vm.run(`module.exports = async function() {${conditionFunction}}()`, __dirname)
             if (typeof response !== 'string') throw new Error('Condition function must return a string')

packages/components/nodes/sequentialagents/CustomFunction/CustomFunction.ts

@@ -102,7 +102,7 @@ class CustomFunction_SeqAgents implements INode {
         if (!sequentialNodes || !sequentialNodes.length) throw new Error('Custom function must have a predecessor!')
 
         const executeFunc = async (state: ISeqAgentsState) => {
-            const variables = await getVars(appDataSource, databaseEntities, nodeData)
+            const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
             const flow = {
                 chatflowId: options.chatflowid,
                 sessionId: options.sessionId,

packages/components/nodes/sequentialagents/ExecuteFlow/ExecuteFlow.ts

@@ -141,7 +141,8 @@ class ExecuteFlow_SeqAgents implements INode {
                 return returnData
             }
 
-            const chatflows = await appDataSource.getRepository(databaseEntities['ChatFlow']).find()
+            const searchOptions = options.searchOptions || {}
+            const chatflows = await appDataSource.getRepository(databaseEntities['ChatFlow']).findBy(searchOptions)
 
             for (let i = 0; i < chatflows.length; i += 1) {
                 const data = {
@@ -189,7 +190,7 @@ class ExecuteFlow_SeqAgents implements INode {
         const chatId = options.chatId
 
         const executeFunc = async (state: ISeqAgentsState) => {
-            const variables = await getVars(appDataSource, databaseEntities, nodeData)
+            const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
 
             let flowInput = ''
             if (seqExecuteFlowInput === 'userQuestion') {
@@ -223,7 +224,7 @@ class ExecuteFlow_SeqAgents implements INode {
                 }
             }
 
-            const options = {
+            const callOptions = {
                 method: 'POST',
                 headers: {
                     'Content-Type': 'application/json',
@@ -234,7 +235,7 @@ class ExecuteFlow_SeqAgents implements INode {
 
             let sandbox: ICommonObject = {
                 $input: flowInput,
-                $callOptions: options,
+                $callOptions: callOptions,
                 $callBody: body,
                 util: undefined,
                 Symbol: undefined,

packages/components/nodes/sequentialagents/LLMNode/LLMNode.ts

@@ -668,7 +668,7 @@ const getReturnOutput = async (nodeData: INodeData, input: string, options: ICom
     const updateStateMemory = nodeData.inputs?.updateStateMemory as string
 
     const selectedTab = tabIdentifier ? tabIdentifier.split(`_${nodeData.id}`)[0] : 'updateStateMemoryUI'
-    const variables = await getVars(appDataSource, databaseEntities, nodeData)
+    const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
 
     const flow = {
         chatflowId: options.chatflowid,
@@ -721,7 +721,7 @@ const getReturnOutput = async (nodeData: INodeData, input: string, options: ICom
             throw new Error(e)
         }
     } else if (selectedTab === 'updateStateMemoryCode' && updateStateMemoryCode) {
-        const vm = await getVM(appDataSource, databaseEntities, nodeData, flow)
+        const vm = await getVM(appDataSource, databaseEntities, nodeData, options, flow)
         try {
             const response = await vm.run(`module.exports = async function() {${updateStateMemoryCode}}()`, __dirname)
             if (typeof response !== 'object') throw new Error('Return output must be an object')

packages/components/nodes/sequentialagents/State/State.ts

@@ -190,7 +190,7 @@ class State_SeqAgents implements INode {
                 throw new Error(e)
             }
         } else if (selectedTab === 'stateMemoryCode' && stateMemoryCode) {
-            const variables = await getVars(appDataSource, databaseEntities, nodeData)
+            const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
             const flow = {
                 chatflowId: options.chatflowid,
                 sessionId: options.sessionId,

packages/components/nodes/sequentialagents/ToolNode/ToolNode.ts

@@ -498,7 +498,7 @@ const getReturnOutput = async (
     const updateStateMemory = nodeData.inputs?.updateStateMemory as string
 
     const selectedTab = tabIdentifier ? tabIdentifier.split(`_${nodeData.id}`)[0] : 'updateStateMemoryUI'
-    const variables = await getVars(appDataSource, databaseEntities, nodeData)
+    const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
 
     const reformattedOutput = outputs.map((output) => {
         return {
@@ -561,7 +561,7 @@ const getReturnOutput = async (
             throw new Error(e)
         }
     } else if (selectedTab === 'updateStateMemoryCode' && updateStateMemoryCode) {
-        const vm = await getVM(appDataSource, databaseEntities, nodeData, flow)
+        const vm = await getVM(appDataSource, databaseEntities, nodeData, options, flow)
         try {
             const response = await vm.run(`module.exports = async function() {${updateStateMemoryCode}}()`, __dirname)
             if (typeof response !== 'object') throw new Error('Return output must be an object')

packages/components/nodes/sequentialagents/commonUtils.ts

@@ -150,8 +150,14 @@ export const processImageMessage = async (llm: BaseChatModel, nodeData: INodeDat
     return multiModalMessageContent
 }
 
-export const getVM = async (appDataSource: DataSource, databaseEntities: IDatabaseEntity, nodeData: INodeData, flow: ICommonObject) => {
-    const variables = await getVars(appDataSource, databaseEntities, nodeData)
+export const getVM = async (
+    appDataSource: DataSource,
+    databaseEntities: IDatabaseEntity,
+    nodeData: INodeData,
+    options: ICommonObject,
+    flow: ICommonObject
+) => {
+    const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
 
     let sandbox: any = {
         util: undefined,
@@ -420,7 +426,7 @@ export const checkMessageHistory = async (
     if (messageHistory) {
         const appDataSource = options.appDataSource as DataSource
         const databaseEntities = options.databaseEntities as IDatabaseEntity
-        const vm = await getVM(appDataSource, databaseEntities, nodeData, {})
+        const vm = await getVM(appDataSource, databaseEntities, nodeData, options, {})
         try {
             const response = await vm.run(`module.exports = async function() {${messageHistory}}()`, __dirname)
             if (!Array.isArray(response)) throw new Error('Returned message history must be an array')