Merge branch 'main' into FEATURE/Vision

# Conflicts:
#	packages/components/nodes/chains/ConversationChain/ConversationChain.ts
#	packages/server/src/index.ts
#	packages/server/src/utils/index.ts

packages/components/src/Interface.ts

@@ -29,6 +29,12 @@ export interface ICommonObject {
     [key: string]: any | CommonType | ICommonObject | CommonType[] | ICommonObject[]
 }
 
+export interface IVariable {
+    name: string
+    value: string
+    type: string
+}
+
 export type IDatabaseEntity = {
     [key: string]: any
 }
@@ -90,7 +96,7 @@ export interface INodeProperties {
     type: string
     icon: string
     version: number
-    category: string
+    category: string // TODO: use enum instead of string
     baseClasses: string[]
     description?: string
     filePath?: string
@@ -139,6 +145,18 @@ export interface IUsedTool {
     toolOutput: string | object
 }
 
+export interface IFileUpload {
+    data?: string
+    type: string
+    name: string
+    mime: string
+}
+
+export interface IMultiModalOption {
+    image?: Record<string, any>
+    audio?: Record<string, any>
+}
+
 /**
  * Classes
  */
@@ -234,10 +252,3 @@ export abstract class FlowiseSummaryMemory extends ConversationSummaryMemory imp
     abstract addChatMessages(msgArray: { text: string; type: MessageType }[], overrideSessionId?: string): Promise<void>
     abstract clearChatMessages(overrideSessionId?: string): Promise<void>
 }
-
-export interface IFileUpload {
-    data: string
-    type: string
-    name: string
-    mime: string
-}

packages/components/src/MultiModalUtils.ts

@@ -1,54 +1,57 @@
-import { ICommonObject, INodeData } from './Interface'
+import { ICommonObject, IFileUpload, IMultiModalOption, INodeData } from './Interface'
 import { BaseChatModel } from 'langchain/chat_models/base'
-import { ChatOpenAI } from 'langchain/chat_models/openai'
+import { ChatOpenAI as LangchainChatOpenAI } from 'langchain/chat_models/openai'
 import path from 'path'
-import { getUserHome } from './utils'
+import { getStoragePath } from './utils'
 import fs from 'fs'
 import { MessageContent } from '@langchain/core/dist/messages'
-import { FlowiseChatOpenAI } from '../nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI'
+import { ChatOpenAI } from '../nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI'
 
 export const injectChainNodeData = (nodeData: INodeData, options: ICommonObject) => {
     let model = nodeData.inputs?.model as BaseChatModel
 
-    if (model instanceof FlowiseChatOpenAI) {
+    if (model instanceof ChatOpenAI) {
         // TODO: this should not be static, need to figure out how to pass the nodeData and options to the invoke method
-        FlowiseChatOpenAI.chainNodeOptions = options
-        FlowiseChatOpenAI.chainNodeData = nodeData
+        ChatOpenAI.chainNodeOptions = options
+        ChatOpenAI.chainNodeData = nodeData
     }
 }
 
-export const addImagesToMessages = (nodeData: INodeData, options: ICommonObject): MessageContent => {
+export const addImagesToMessages = (nodeData: INodeData, options: ICommonObject, multiModalOption?: IMultiModalOption): MessageContent => {
     const imageContent: MessageContent = []
-    let model = nodeData.inputs?.model as BaseChatModel
-    if (model instanceof ChatOpenAI && (model as any).multiModal) {
-        if (options?.uploads && options?.uploads.length > 0) {
+    let model = nodeData.inputs?.model
+
+    if (model instanceof LangchainChatOpenAI && multiModalOption) {
+        // Image Uploaded
+        if (multiModalOption.image && multiModalOption.image.allowImageUploads && options?.uploads && options?.uploads.length > 0) {
             const imageUploads = getImageUploads(options.uploads)
             for (const upload of imageUploads) {
                 let bf = upload.data
                 if (upload.type == 'stored-file') {
-                    const filePath = path.join(getUserHome(), '.flowise', 'gptvision', upload.data, upload.name)
+                    const filePath = path.join(getStoragePath(), options.chatflowid, options.chatId, upload.name)
 
                     // as the image is stored in the server, read the file and convert it to base64
                     const contents = fs.readFileSync(filePath)
                     bf = 'data:' + upload.mime + ';base64,' + contents.toString('base64')
+
+                    imageContent.push({
+                        type: 'image_url',
+                        image_url: {
+                            url: bf,
+                            detail: multiModalOption.image.imageResolution ?? 'low'
+                        }
+                    })
                 }
-                imageContent.push({
-                    type: 'image_url',
-                    image_url: {
-                        url: bf,
-                        detail: 'low'
-                    }
-                })
             }
         }
     }
     return imageContent
 }
 
-export const getAudioUploads = (uploads: any[]) => {
-    return uploads.filter((url: any) => url.mime.startsWith('audio/'))
+export const getAudioUploads = (uploads: IFileUpload[]) => {
+    return uploads.filter((upload: IFileUpload) => upload.mime.startsWith('audio/'))
 }
 
-export const getImageUploads = (uploads: any[]) => {
-    return uploads.filter((url: any) => url.mime.startsWith('image/'))
+export const getImageUploads = (uploads: IFileUpload[]) => {
+    return uploads.filter((upload: IFileUpload) => upload.mime.startsWith('image/'))
 }

packages/components/src/agents.ts

@@ -1,5 +1,6 @@
+import { flatten } from 'lodash'
 import { AgentExecutorInput, BaseSingleActionAgent, BaseMultiActionAgent, RunnableAgent, StoppingMethod } from 'langchain/agents'
-import { ChainValues, AgentStep, AgentFinish, AgentAction, BaseMessage, FunctionMessage, AIMessage } from 'langchain/schema'
+import { ChainValues, AgentStep, AgentAction, BaseMessage, FunctionMessage, AIMessage } from 'langchain/schema'
 import { OutputParserException } from 'langchain/schema/output_parser'
 import { CallbackManager, CallbackManagerForChainRun, Callbacks } from 'langchain/callbacks'
 import { ToolInputParsingException, Tool } from '@langchain/core/tools'
@@ -7,6 +8,11 @@ import { Runnable } from 'langchain/schema/runnable'
 import { BaseChain, SerializedLLMChain } from 'langchain/chains'
 import { Serializable } from '@langchain/core/load/serializable'
 
+export const SOURCE_DOCUMENTS_PREFIX = '\n\n----FLOWISE_SOURCE_DOCUMENTS----\n\n'
+type AgentFinish = {
+    returnValues: Record<string, any>
+    log: string
+}
 type AgentExecutorOutput = ChainValues
 
 interface AgentExecutorIteratorInput {
@@ -315,10 +321,12 @@ export class AgentExecutor extends BaseChain<ChainValues, AgentExecutorOutput> {
 
         const steps: AgentStep[] = []
         let iterations = 0
+        let sourceDocuments: Array<Document> = []
 
         const getOutput = async (finishStep: AgentFinish): Promise<AgentExecutorOutput> => {
             const { returnValues } = finishStep
             const additional = await this.agent.prepareForOutput(returnValues, steps)
+            if (sourceDocuments.length) additional.sourceDocuments = flatten(sourceDocuments)
 
             if (this.returnIntermediateSteps) {
                 return { ...returnValues, intermediateSteps: steps, ...additional }
@@ -406,6 +414,17 @@ export class AgentExecutor extends BaseChain<ChainValues, AgentExecutorOutput> {
                             return { action, observation: observation ?? '' }
                         }
                     }
+                    if (observation?.includes(SOURCE_DOCUMENTS_PREFIX)) {
+                        const observationArray = observation.split(SOURCE_DOCUMENTS_PREFIX)
+                        observation = observationArray[0]
+                        const docs = observationArray[1]
+                        try {
+                            const parsedDocs = JSON.parse(docs)
+                            sourceDocuments.push(parsedDocs)
+                        } catch (e) {
+                            console.error('Error parsing source documents from tool')
+                        }
+                    }
                     return { action, observation: observation ?? '' }
                 })
             )
@@ -500,6 +519,10 @@ export class AgentExecutor extends BaseChain<ChainValues, AgentExecutorOutput> {
                         chatId: this.chatId,
                         input: this.input
                     })
+                    if (observation?.includes(SOURCE_DOCUMENTS_PREFIX)) {
+                        const observationArray = observation.split(SOURCE_DOCUMENTS_PREFIX)
+                        observation = observationArray[0]
+                    }
                 } catch (e) {
                     if (e instanceof ToolInputParsingException) {
                         if (this.handleParsingErrors === true) {

packages/components/src/speechToText.ts

@@ -1,17 +1,16 @@
-import { ICommonObject } from './Interface'
-import { getCredentialData, getUserHome } from './utils'
+import { ICommonObject, IFileUpload } from './Interface'
+import { getCredentialData, getStoragePath } from './utils'
 import { type ClientOptions, OpenAIClient } from '@langchain/openai'
 import fs from 'fs'
 import path from 'path'
 import { AssemblyAI } from 'assemblyai'
 
-export const convertSpeechToText = async (upload: any, speechToTextConfig: any, options: ICommonObject) => {
+export const convertSpeechToText = async (upload: IFileUpload, speechToTextConfig: ICommonObject, options: ICommonObject) => {
     if (speechToTextConfig) {
         const credentialId = speechToTextConfig.credentialId as string
         const credentialData = await getCredentialData(credentialId ?? '', options)
-        const filePath = path.join(getUserHome(), '.flowise', 'gptvision', upload.data, upload.name)
+        const filePath = path.join(getStoragePath(), options.chatflowid, options.chatId, upload.name)
 
-        // as the image is stored in the server, read the file and convert it to base64
         const audio_file = fs.createReadStream(filePath)
 
         if (speechToTextConfig.name === 'openAIWhisper') {

packages/components/src/utils.ts

@@ -5,7 +5,7 @@ import * as path from 'path'
 import { JSDOM } from 'jsdom'
 import { z } from 'zod'
 import { DataSource } from 'typeorm'
-import { ICommonObject, IDatabaseEntity, IMessage, INodeData } from './Interface'
+import { ICommonObject, IDatabaseEntity, IMessage, INodeData, IVariable } from './Interface'
 import { AES, enc } from 'crypto-js'
 import { ChatMessageHistory } from 'langchain/memory'
 import { AIMessage, HumanMessage, BaseMessage } from 'langchain/schema'
@@ -70,6 +70,22 @@ export const availableDependencies = [
     'weaviate-ts-client'
 ]
 
+export const defaultAllowBuiltInDep = [
+    'assert',
+    'buffer',
+    'crypto',
+    'events',
+    'http',
+    'https',
+    'net',
+    'path',
+    'querystring',
+    'timers',
+    'tls',
+    'url',
+    'zlib'
+]
+
 /**
  * Get base classes of components
  *
@@ -673,3 +689,79 @@ export const convertBaseMessagetoIMessage = (messages: BaseMessage[]): IMessage[
     }
     return formatmessages
 }
+
+/**
+ * Convert MultiOptions String to String Array
+ * @param {string} inputString
+ * @returns {string[]}
+ */
+export const convertMultiOptionsToStringArray = (inputString: string): string[] => {
+    let ArrayString: string[] = []
+    try {
+        ArrayString = JSON.parse(inputString)
+    } catch (e) {
+        ArrayString = []
+    }
+    return ArrayString
+}
+
+/**
+ * Get variables
+ * @param {DataSource} appDataSource
+ * @param {IDatabaseEntity} databaseEntities
+ * @param {INodeData} nodeData
+ */
+export const getVars = async (appDataSource: DataSource, databaseEntities: IDatabaseEntity, nodeData: INodeData) => {
+    const variables = ((await appDataSource.getRepository(databaseEntities['Variable']).find()) as IVariable[]) ?? []
+
+    // override variables defined in overrideConfig
+    // nodeData.inputs.variables is an Object, check each property and override the variable
+    if (nodeData?.inputs?.vars) {
+        for (const propertyName of Object.getOwnPropertyNames(nodeData.inputs.vars)) {
+            const foundVar = variables.find((v) => v.name === propertyName)
+            if (foundVar) {
+                // even if the variable was defined as runtime, we override it with static value
+                foundVar.type = 'static'
+                foundVar.value = nodeData.inputs.vars[propertyName]
+            } else {
+                // add it the variables, if not found locally in the db
+                variables.push({ name: propertyName, type: 'static', value: nodeData.inputs.vars[propertyName] })
+            }
+        }
+    }
+
+    return variables
+}
+
+/**
+ * Prepare sandbox variables
+ * @param {IVariable[]} variables
+ */
+export const prepareSandboxVars = (variables: IVariable[]) => {
+    let vars = {}
+    if (variables) {
+        for (const item of variables) {
+            let value = item.value
+
+            // read from .env file
+            if (item.type === 'runtime') {
+                value = process.env[item.name] ?? ''
+            }
+
+            Object.defineProperty(vars, item.name, {
+                enumerable: true,
+                configurable: true,
+                writable: true,
+                value: value
+            })
+        }
+    }
+    return vars
+}
+
+/**
+ * Prepare storage path
+ */
+export const getStoragePath = (): string => {
+    return process.env.BLOB_STORAGE_PATH ? path.join(process.env.BLOB_STORAGE_PATH) : path.join(getUserHome(), '.flowise', 'storage')
+}